{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>City</th>\n",
       "      <th>Monthly_Income</th>\n",
       "      <th>Loan_Amount_Applied</th>\n",
       "      <th>Loan_Tenure_Applied</th>\n",
       "      <th>Existing_EMI</th>\n",
       "      <th>Employer_Name</th>\n",
       "      <th>Salary_Account</th>\n",
       "      <th>Mobile_Verified</th>\n",
       "      <th>Var5</th>\n",
       "      <th>...</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>EMI_Loan_Submitted</th>\n",
       "      <th>Filled_Form</th>\n",
       "      <th>Device_Type</th>\n",
       "      <th>Var2</th>\n",
       "      <th>Source</th>\n",
       "      <th>Var4</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>Employer_Name_Count</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>172</td>\n",
       "      <td>-0.017842</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>8693</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.263100</td>\n",
       "      <td>0.624487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>446</td>\n",
       "      <td>-0.010953</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>38692</td>\n",
       "      <td>22</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.038911</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.222116</td>\n",
       "      <td>0.185533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>497</td>\n",
       "      <td>-0.016693</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1835</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.263100</td>\n",
       "      <td>0.424564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>568</td>\n",
       "      <td>-0.010953</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>5700</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.261333</td>\n",
       "      <td>0.059120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>87</td>\n",
       "      <td>0.018898</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0025</td>\n",
       "      <td>13515</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.260449</td>\n",
       "      <td>0.285170</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Gender  City  Monthly_Income  Loan_Amount_Applied  Loan_Tenure_Applied  \\\n",
       "0       0   172       -0.017842                 0.03                  0.5   \n",
       "1       1   446       -0.010953                 0.02                  0.2   \n",
       "2       1   497       -0.016693                 0.06                  0.4   \n",
       "3       1   568       -0.010953                 0.10                  0.5   \n",
       "4       1    87        0.018898                 0.05                  0.2   \n",
       "\n",
       "   Existing_EMI  Employer_Name  Salary_Account  Mobile_Verified  Var5  ...  \\\n",
       "0        0.0000           8693              20                1     0  ...   \n",
       "1        0.0000          38692              22                2     5  ...   \n",
       "2        0.0000           1835              45                2     0  ...   \n",
       "3        0.0000           5700              45                2     2  ...   \n",
       "4        0.0025          13515              20                2     9  ...   \n",
       "\n",
       "   Processing_Fee  EMI_Loan_Submitted  Filled_Form  Device_Type  Var2  Source  \\\n",
       "0             NaN                 NaN            1            2     6       1   \n",
       "1             NaN            0.038911            1            2     6       1   \n",
       "2             NaN                 NaN            1            2     1      17   \n",
       "3             NaN                 NaN            1            2     1      17   \n",
       "4             NaN                 NaN            1            2     1       9   \n",
       "\n",
       "   Var4  Disbursed  Employer_Name_Count       age  \n",
       "0     1          0            -0.263100  0.624487  \n",
       "1     3          0             0.222116  0.185533  \n",
       "2     1          0            -0.263100  0.424564  \n",
       "3     3          0            -0.261333  0.059120  \n",
       "4     3          0            -0.260449  0.285170  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_type={'Gender': np.int8,'City': np.int16,'Monthly_Income':np.float64,'Loan_Amount_Applied': np.float64\n",
    "          ,'Loan_Tenure_Applied': np.float64,'Existing_EMI':np.float64,'Employer_Name':np.int32,'Salary_Account':np.int8\n",
    "          ,'Mobile_Verified':np.int8,'Var5':np.int8,'Var1':np.int8,'Loan_Amount_Submitted':np.float64,'Loan_Tenure_Submitted':np.float64\n",
    "          ,'Interest_Rate':np.float64,'Processing_Fee':np.float64,'EMI_Loan_Submitted':np.float64\n",
    "          ,'Filled_Form':np.int8,'Device_Type':np.int8,'Var2':np.int8,'Source':np.int8,'Var4':np.int8,'Disbursed':np.int64\n",
    "          ,'Employer_Name_Count':np.float64,'age':np.float64}\n",
    "\n",
    "train = pd.read_csv(\"FE_predict-customer-worth-for-happy-customer-bank.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 87020 entries, 0 to 87019\n",
      "Data columns (total 24 columns):\n",
      "Gender                   87020 non-null int64\n",
      "City                     87020 non-null int64\n",
      "Monthly_Income           87020 non-null float64\n",
      "Loan_Amount_Applied      86949 non-null float64\n",
      "Loan_Tenure_Applied      86949 non-null float64\n",
      "Existing_EMI             86949 non-null float64\n",
      "Employer_Name            87020 non-null int64\n",
      "Salary_Account           87020 non-null int64\n",
      "Mobile_Verified          87020 non-null int64\n",
      "Var5                     87020 non-null int64\n",
      "Var1                     87020 non-null int64\n",
      "Loan_Amount_Submitted    52407 non-null float64\n",
      "Loan_Tenure_Submitted    52407 non-null float64\n",
      "Interest_Rate            27726 non-null float64\n",
      "Processing_Fee           27420 non-null float64\n",
      "EMI_Loan_Submitted       27726 non-null float64\n",
      "Filled_Form              87020 non-null int64\n",
      "Device_Type              87020 non-null int64\n",
      "Var2                     87020 non-null int64\n",
      "Source                   87020 non-null int64\n",
      "Var4                     87020 non-null int64\n",
      "Disbursed                87020 non-null int64\n",
      "Employer_Name_Count      86949 non-null float64\n",
      "age                      87020 non-null float64\n",
      "dtypes: float64(11), int64(13)\n",
      "memory usage: 15.9 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>City</th>\n",
       "      <th>Monthly_Income</th>\n",
       "      <th>Loan_Amount_Applied</th>\n",
       "      <th>Loan_Tenure_Applied</th>\n",
       "      <th>Existing_EMI</th>\n",
       "      <th>Employer_Name</th>\n",
       "      <th>Salary_Account</th>\n",
       "      <th>Mobile_Verified</th>\n",
       "      <th>Var5</th>\n",
       "      <th>...</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>EMI_Loan_Submitted</th>\n",
       "      <th>Filled_Form</th>\n",
       "      <th>Device_Type</th>\n",
       "      <th>Var2</th>\n",
       "      <th>Source</th>\n",
       "      <th>Var4</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>Employer_Name_Count</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>8.702000e+04</td>\n",
       "      <td>86949.000000</td>\n",
       "      <td>86949.000000</td>\n",
       "      <td>86949.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>27420.000000</td>\n",
       "      <td>27726.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>8.694900e+04</td>\n",
       "      <td>8.702000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.572834</td>\n",
       "      <td>283.649000</td>\n",
       "      <td>8.165281e-19</td>\n",
       "      <td>0.023025</td>\n",
       "      <td>0.213140</td>\n",
       "      <td>0.000370</td>\n",
       "      <td>20097.750333</td>\n",
       "      <td>21.058630</td>\n",
       "      <td>1.649046</td>\n",
       "      <td>4.922167</td>\n",
       "      <td>...</td>\n",
       "      <td>0.099019</td>\n",
       "      <td>0.068420</td>\n",
       "      <td>1.223960</td>\n",
       "      <td>1.739083</td>\n",
       "      <td>3.146116</td>\n",
       "      <td>7.228476</td>\n",
       "      <td>2.949793</td>\n",
       "      <td>0.014629</td>\n",
       "      <td>-3.922535e-18</td>\n",
       "      <td>-2.122973e-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.494670</td>\n",
       "      <td>180.639053</td>\n",
       "      <td>1.000006e+00</td>\n",
       "      <td>0.035421</td>\n",
       "      <td>0.201419</td>\n",
       "      <td>0.003981</td>\n",
       "      <td>13242.275750</td>\n",
       "      <td>16.139054</td>\n",
       "      <td>0.477295</td>\n",
       "      <td>5.622229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.094896</td>\n",
       "      <td>0.052324</td>\n",
       "      <td>0.416926</td>\n",
       "      <td>0.439163</td>\n",
       "      <td>2.304923</td>\n",
       "      <td>7.713886</td>\n",
       "      <td>1.697736</td>\n",
       "      <td>0.120062</td>\n",
       "      <td>1.000006e+00</td>\n",
       "      <td>1.000006e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-2.702641e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.631004e-01</td>\n",
       "      <td>-4.744392e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>-1.944891e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8548.750000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.036145</td>\n",
       "      <td>0.037021</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.631004e-01</td>\n",
       "      <td>-5.723120e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>269.000000</td>\n",
       "      <td>-1.554535e-02</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18668.500000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.076305</td>\n",
       "      <td>0.057230</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.622166e-01</td>\n",
       "      <td>1.336046e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>446.000000</td>\n",
       "      <td>-8.656709e-03</td>\n",
       "      <td>0.030000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000350</td>\n",
       "      <td>31595.250000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.121486</td>\n",
       "      <td>0.081789</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.507269e-01</td>\n",
       "      <td>3.734475e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>696.000000</td>\n",
       "      <td>2.041313e+02</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>43566.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.079104e+00</td>\n",
       "      <td>1.193425e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Gender          City  Monthly_Income  Loan_Amount_Applied  \\\n",
       "count  87020.000000  87020.000000    8.702000e+04         86949.000000   \n",
       "mean       0.572834    283.649000    8.165281e-19             0.023025   \n",
       "std        0.494670    180.639053    1.000006e+00             0.035421   \n",
       "min        0.000000     -1.000000   -2.702641e-02             0.000000   \n",
       "25%        0.000000    134.000000   -1.944891e-02             0.000000   \n",
       "50%        1.000000    269.000000   -1.554535e-02             0.010000   \n",
       "75%        1.000000    446.000000   -8.656709e-03             0.030000   \n",
       "max        1.000000    696.000000    2.041313e+02             1.000000   \n",
       "\n",
       "       Loan_Tenure_Applied  Existing_EMI  Employer_Name  Salary_Account  \\\n",
       "count         86949.000000  86949.000000   87020.000000    87020.000000   \n",
       "mean              0.213140      0.000370   20097.750333       21.058630   \n",
       "std               0.201419      0.003981   13242.275750       16.139054   \n",
       "min               0.000000      0.000000      -1.000000       -1.000000   \n",
       "25%               0.000000      0.000000    8548.750000        5.000000   \n",
       "50%               0.200000      0.000000   18668.500000       20.000000   \n",
       "75%               0.400000      0.000350   31595.250000       32.000000   \n",
       "max               1.000000      1.000000   43566.000000       57.000000   \n",
       "\n",
       "       Mobile_Verified          Var5  ...  Processing_Fee  EMI_Loan_Submitted  \\\n",
       "count     87020.000000  87020.000000  ...    27420.000000        27726.000000   \n",
       "mean          1.649046      4.922167  ...        0.099019            0.068420   \n",
       "std           0.477295      5.622229  ...        0.094896            0.052324   \n",
       "min           0.000000      0.000000  ...        0.000000            0.000000   \n",
       "25%           1.000000      0.000000  ...        0.036145            0.037021   \n",
       "50%           2.000000      2.000000  ...        0.076305            0.057230   \n",
       "75%           2.000000      9.000000  ...        0.121486            0.081789   \n",
       "max           2.000000     19.000000  ...        1.000000            1.000000   \n",
       "\n",
       "        Filled_Form   Device_Type          Var2        Source          Var4  \\\n",
       "count  87020.000000  87020.000000  87020.000000  87020.000000  87020.000000   \n",
       "mean       1.223960      1.739083      3.146116      7.228476      2.949793   \n",
       "std        0.416926      0.439163      2.304923      7.713886      1.697736   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "50%        1.000000      2.000000      2.000000      8.000000      3.000000   \n",
       "75%        1.000000      2.000000      6.000000      8.000000      5.000000   \n",
       "max        2.000000      2.000000      7.000000     30.000000      7.000000   \n",
       "\n",
       "          Disbursed  Employer_Name_Count           age  \n",
       "count  87020.000000         8.694900e+04  8.702000e+04  \n",
       "mean       0.014629        -3.922535e-18 -2.122973e-18  \n",
       "std        0.120062         1.000006e+00  1.000006e+00  \n",
       "min        0.000000        -2.631004e-01 -4.744392e+00  \n",
       "25%        0.000000        -2.631004e-01 -5.723120e-02  \n",
       "50%        0.000000        -2.622166e-01  1.336046e-01  \n",
       "75%        0.000000        -2.507269e-01  3.734475e-01  \n",
       "max        1.000000         4.079104e+00  1.193425e+00  \n",
       "\n",
       "[8 rows x 24 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['Gender']=train['Gender'].astype(np.int8) #类别型特征，必须是less than int32的整数\n",
    "train['City']=train['City'].astype(np.int16) #负数作为类别型特征将视为缺省特征\n",
    "train['Employer_Name']=train['Employer_Name'].astype(np.int32) #类别型特征，必须是less than int32的整数,负数将视为缺省特征\n",
    "train['Salary_Account']=train['Salary_Account'].astype(np.int8)\n",
    "train['Mobile_Verified']=train['Mobile_Verified'].astype(np.int8)\n",
    "train['Var5']=train['Var5'].astype(np.int8)\n",
    "train['Var1']=train['Var1'].astype(np.int8)\n",
    "train['Filled_Form']=train['Filled_Form'].astype(np.int8)\n",
    "train['Device_Type']=train['Device_Type'].astype(np.int8)\n",
    "train['Var2']=train['Var2'].astype(np.int8)\n",
    "train['Source']=train['Source'].astype(np.int8)\n",
    "train['Var4']=train['Var4'].astype(np.int8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 87020 entries, 0 to 87019\n",
      "Data columns (total 24 columns):\n",
      "Gender                   87020 non-null int8\n",
      "City                     87020 non-null int16\n",
      "Monthly_Income           87020 non-null float64\n",
      "Loan_Amount_Applied      86949 non-null float64\n",
      "Loan_Tenure_Applied      86949 non-null float64\n",
      "Existing_EMI             86949 non-null float64\n",
      "Employer_Name            87020 non-null int32\n",
      "Salary_Account           87020 non-null int8\n",
      "Mobile_Verified          87020 non-null int8\n",
      "Var5                     87020 non-null int8\n",
      "Var1                     87020 non-null int8\n",
      "Loan_Amount_Submitted    52407 non-null float64\n",
      "Loan_Tenure_Submitted    52407 non-null float64\n",
      "Interest_Rate            27726 non-null float64\n",
      "Processing_Fee           27420 non-null float64\n",
      "EMI_Loan_Submitted       27726 non-null float64\n",
      "Filled_Form              87020 non-null int8\n",
      "Device_Type              87020 non-null int8\n",
      "Var2                     87020 non-null int8\n",
      "Source                   87020 non-null int8\n",
      "Var4                     87020 non-null int8\n",
      "Disbursed                87020 non-null int64\n",
      "Employer_Name_Count      86949 non-null float64\n",
      "age                      87020 non-null float64\n",
      "dtypes: float64(11), int16(1), int32(1), int64(1), int8(10)\n",
      "memory usage: 9.3 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Gender</th>\n",
       "      <th>City</th>\n",
       "      <th>Monthly_Income</th>\n",
       "      <th>Loan_Amount_Applied</th>\n",
       "      <th>Loan_Tenure_Applied</th>\n",
       "      <th>Existing_EMI</th>\n",
       "      <th>Employer_Name</th>\n",
       "      <th>Salary_Account</th>\n",
       "      <th>Mobile_Verified</th>\n",
       "      <th>Var5</th>\n",
       "      <th>...</th>\n",
       "      <th>Processing_Fee</th>\n",
       "      <th>EMI_Loan_Submitted</th>\n",
       "      <th>Filled_Form</th>\n",
       "      <th>Device_Type</th>\n",
       "      <th>Var2</th>\n",
       "      <th>Source</th>\n",
       "      <th>Var4</th>\n",
       "      <th>Disbursed</th>\n",
       "      <th>Employer_Name_Count</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>8.702000e+04</td>\n",
       "      <td>86949.000000</td>\n",
       "      <td>86949.000000</td>\n",
       "      <td>86949.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>27420.000000</td>\n",
       "      <td>27726.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>87020.000000</td>\n",
       "      <td>8.694900e+04</td>\n",
       "      <td>8.702000e+04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.572834</td>\n",
       "      <td>283.649000</td>\n",
       "      <td>8.165281e-19</td>\n",
       "      <td>0.023025</td>\n",
       "      <td>0.213140</td>\n",
       "      <td>0.000370</td>\n",
       "      <td>20097.750333</td>\n",
       "      <td>21.058630</td>\n",
       "      <td>1.649046</td>\n",
       "      <td>4.922167</td>\n",
       "      <td>...</td>\n",
       "      <td>0.099019</td>\n",
       "      <td>0.068420</td>\n",
       "      <td>1.223960</td>\n",
       "      <td>1.739083</td>\n",
       "      <td>3.146116</td>\n",
       "      <td>7.228476</td>\n",
       "      <td>2.949793</td>\n",
       "      <td>0.014629</td>\n",
       "      <td>-3.922535e-18</td>\n",
       "      <td>-2.122973e-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.494670</td>\n",
       "      <td>180.639053</td>\n",
       "      <td>1.000006e+00</td>\n",
       "      <td>0.035421</td>\n",
       "      <td>0.201419</td>\n",
       "      <td>0.003981</td>\n",
       "      <td>13242.275750</td>\n",
       "      <td>16.139054</td>\n",
       "      <td>0.477295</td>\n",
       "      <td>5.622229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.094896</td>\n",
       "      <td>0.052324</td>\n",
       "      <td>0.416926</td>\n",
       "      <td>0.439163</td>\n",
       "      <td>2.304923</td>\n",
       "      <td>7.713886</td>\n",
       "      <td>1.697736</td>\n",
       "      <td>0.120062</td>\n",
       "      <td>1.000006e+00</td>\n",
       "      <td>1.000006e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-2.702641e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.631004e-01</td>\n",
       "      <td>-4.744392e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>134.000000</td>\n",
       "      <td>-1.944891e-02</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>8548.750000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.036145</td>\n",
       "      <td>0.037021</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.631004e-01</td>\n",
       "      <td>-5.723120e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>269.000000</td>\n",
       "      <td>-1.554535e-02</td>\n",
       "      <td>0.010000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>18668.500000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.076305</td>\n",
       "      <td>0.057230</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.622166e-01</td>\n",
       "      <td>1.336046e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>446.000000</td>\n",
       "      <td>-8.656709e-03</td>\n",
       "      <td>0.030000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000350</td>\n",
       "      <td>31595.250000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.121486</td>\n",
       "      <td>0.081789</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-2.507269e-01</td>\n",
       "      <td>3.734475e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>696.000000</td>\n",
       "      <td>2.041313e+02</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>43566.000000</td>\n",
       "      <td>57.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.079104e+00</td>\n",
       "      <td>1.193425e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Gender          City  Monthly_Income  Loan_Amount_Applied  \\\n",
       "count  87020.000000  87020.000000    8.702000e+04         86949.000000   \n",
       "mean       0.572834    283.649000    8.165281e-19             0.023025   \n",
       "std        0.494670    180.639053    1.000006e+00             0.035421   \n",
       "min        0.000000     -1.000000   -2.702641e-02             0.000000   \n",
       "25%        0.000000    134.000000   -1.944891e-02             0.000000   \n",
       "50%        1.000000    269.000000   -1.554535e-02             0.010000   \n",
       "75%        1.000000    446.000000   -8.656709e-03             0.030000   \n",
       "max        1.000000    696.000000    2.041313e+02             1.000000   \n",
       "\n",
       "       Loan_Tenure_Applied  Existing_EMI  Employer_Name  Salary_Account  \\\n",
       "count         86949.000000  86949.000000   87020.000000    87020.000000   \n",
       "mean              0.213140      0.000370   20097.750333       21.058630   \n",
       "std               0.201419      0.003981   13242.275750       16.139054   \n",
       "min               0.000000      0.000000      -1.000000       -1.000000   \n",
       "25%               0.000000      0.000000    8548.750000        5.000000   \n",
       "50%               0.200000      0.000000   18668.500000       20.000000   \n",
       "75%               0.400000      0.000350   31595.250000       32.000000   \n",
       "max               1.000000      1.000000   43566.000000       57.000000   \n",
       "\n",
       "       Mobile_Verified          Var5  ...  Processing_Fee  EMI_Loan_Submitted  \\\n",
       "count     87020.000000  87020.000000  ...    27420.000000        27726.000000   \n",
       "mean          1.649046      4.922167  ...        0.099019            0.068420   \n",
       "std           0.477295      5.622229  ...        0.094896            0.052324   \n",
       "min           0.000000      0.000000  ...        0.000000            0.000000   \n",
       "25%           1.000000      0.000000  ...        0.036145            0.037021   \n",
       "50%           2.000000      2.000000  ...        0.076305            0.057230   \n",
       "75%           2.000000      9.000000  ...        0.121486            0.081789   \n",
       "max           2.000000     19.000000  ...        1.000000            1.000000   \n",
       "\n",
       "        Filled_Form   Device_Type          Var2        Source          Var4  \\\n",
       "count  87020.000000  87020.000000  87020.000000  87020.000000  87020.000000   \n",
       "mean       1.223960      1.739083      3.146116      7.228476      2.949793   \n",
       "std        0.416926      0.439163      2.304923      7.713886      1.697736   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "50%        1.000000      2.000000      2.000000      8.000000      3.000000   \n",
       "75%        1.000000      2.000000      6.000000      8.000000      5.000000   \n",
       "max        2.000000      2.000000      7.000000     30.000000      7.000000   \n",
       "\n",
       "          Disbursed  Employer_Name_Count           age  \n",
       "count  87020.000000         8.694900e+04  8.702000e+04  \n",
       "mean       0.014629        -3.922535e-18 -2.122973e-18  \n",
       "std        0.120062         1.000006e+00  1.000006e+00  \n",
       "min        0.000000        -2.631004e-01 -4.744392e+00  \n",
       "25%        0.000000        -2.631004e-01 -5.723120e-02  \n",
       "50%        0.000000        -2.622166e-01  1.336046e-01  \n",
       "75%        0.000000        -2.507269e-01  3.734475e-01  \n",
       "max        1.000000         4.079104e+00  1.193425e+00  \n",
       "\n",
       "[8 rows x 24 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train = train['Disbursed'] #标签\n",
    "X_train = train.drop([\"Disbursed\"], axis=1) #训练样本\n",
    "\n",
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "categorical_feat=['Gender','City','Employer_Name','Salary_Account','Mobile_Verified','Var5','Var1','Filled_Form','Device_Type','Var2','Source','Var4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<lightgbm.basic.Dataset at 0x7f0321224400>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbmtrain = lgbm.Dataset(X_train, Y_train,categorical_feature=categorical_feat)\n",
    "lgbmtrain"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1)选择基学习器及其个数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "goss - n_estimators: 754\n",
      "goss - cv score: 0.09344239429058451\n",
      "gbdt - n_estimators: 628\n",
      "gbdt - cv score: 0.09581963922985523\n",
      "random_forest- n_estimators: 1\n",
      "random_forest- cv score: 13.728207146552453\n"
     ]
    }
   ],
   "source": [
    "boostings = ('goss', 'gbdt')\n",
    "for i,v in enumerate(boostings):\n",
    "    lgbm_params = {'boosting': v,'objective': 'binary','n_jobs': 5,'is_unbalance':True}\n",
    "    cv_result = lgbm.cv(lgbm_params,lgbmtrain,num_boost_round=10000,nfold=5,metrics='binary_logloss',early_stopping_rounds=10,categorical_feature=categorical_feat,seed=7)\n",
    "    print(v,'- n_estimators:' , len(cv_result['binary_logloss-mean']))\n",
    "    print(v,'- cv score:' , cv_result['binary_logloss-mean'][-1])\n",
    "\n",
    "lgbm_params = {'boosting': 'random_forest','objective': 'binary','n_jobs': 5,'is_unbalance':True,'bagging_fraction':0.7,'bagging_freq':1}\n",
    "cv_result = lgbm.cv(lgbm_params,lgbmtrain,num_boost_round=10000,nfold=5,metrics='binary_logloss',early_stopping_rounds=10,categorical_feature=categorical_feat,seed=7)\n",
    "print('random_forest- n_estimators:' , len(cv_result['binary_logloss-mean']))\n",
    "print('random_forest- cv score:' , cv_result['binary_logloss-mean'][-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上可见： Gradient-based One-Side Sampling (基于梯度的单侧采样)需要741棵树，而gbdt（传统的梯度提升决策树）需要628棵树，比goss简单，Random Forest (随机森林)只需1棵树，明细不合理，且损失大，而gbdt损失较小，所以最佳基学习器为gbdt，个数暂定628个"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2)num_leaves & max_depth\n",
    "\n",
    "#num_leaves建议70-80，搜索区间50-80,值越大模型越复杂，越容易过拟合 相应的扩大max_depth\n",
    "#max_depth常用3-10。这个值为树的最大深度。这个值是用来控制过拟合的。max_depth越大，模型学习的更加具体。设置为0代表没有限制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['explained_variance', 'r2', 'neg_median_absolute_error', 'neg_mean_absolute_error', 'neg_mean_squared_error', 'neg_mean_squared_log_error', 'accuracy', 'roc_auc', 'balanced_accuracy', 'average_precision', 'neg_log_loss', 'brier_score_loss', 'adjusted_rand_score', 'homogeneity_score', 'completeness_score', 'v_measure_score', 'mutual_info_score', 'adjusted_mutual_info_score', 'normalized_mutual_info_score', 'fowlkes_mallows_score', 'precision', 'precision_macro', 'precision_micro', 'precision_samples', 'precision_weighted', 'recall', 'recall_macro', 'recall_micro', 'recall_samples', 'recall_weighted', 'f1', 'f1_macro', 'f1_micro', 'f1_samples', 'f1_weighted'])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn\n",
    "a=sklearn.metrics.SCORERS.keys()\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   48.7s\n",
      "[Parallel(n_jobs=5)]: Done  62 tasks      | elapsed:  7.9min\n",
      "[Parallel(n_jobs=5)]: Done  80 out of  80 | elapsed: 10.8min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=-1, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=628,\n",
       "        n_jobs=5, num_leaves=31, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'num_leaves': range(50, 90, 10), 'max_depth': range(3, 10, 2)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=7)\n",
    "\n",
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(50,90,10) #50,60,70,80\n",
    "max_depth_s = range(3,10,2) #3,5,7,9\n",
    "\n",
    "tuned_parameters = dict( num_leaves = num_leaves_s , max_depth = max_depth_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.09353734098946158\n",
      "{'max_depth': 9, 'num_leaves': 50}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAELCAYAAAARNxsIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4VHX2x/H3ycykA6GLdBBBBQSMoqhYQUVcBARXXVd3USxY0B/2spZVUde+NrDrrqgIgooiICoWYEF6Dz0htAABEkgyk/P7Yy4aYgIZmMmdSc7reeaZ2+dzc2HO3Pa9oqoYY4wx4RDndgBjjDFVhxUVY4wxYWNFxRhjTNhYUTHGGBM2VlSMMcaEjRUVY4wxYWNFxRhjTNhYUTHGGBM2VlSMMcaEjdftAJWtXr162qJFC7djGGNMTJk9e/ZWVa1/sOmqXVFp0aIFs2bNcjuGMcbEFBFZW5Hp7PCXMcaYsLGiYowxJmysqBhjjAmbandOpSxFRUVkZmayd+9et6OUKzExkSZNmuDz+dyOYowx5bKiAmRmZlKjRg1atGiBiLgd5w9UlZycHDIzM2nZsqXbcYwxplx2+AvYu3cvdevWjcqCAiAi1K1bN6r3pIwxBqyo/CZaC8o+0Z7PGGPADn8ZY0zMKy5W9hQFyCv0k1/gvBcGyCvY//3yrs3weSK7L2FFJQrs3buX7t27U1BQgN/v55JLLuHhhx92O5YxJgL8gWLyiwK/f/n/VgT85BUE9n8vDJBf4LyXGL671HT5hYEKfXafTkeSlhwf0fWzohIFEhIS+Pbbb0lNTaWoqIjTTjuNCy64gJNPPtntaMZUa0WBYvILAuwu9P/+5V7Gl/zvRaCcPYUSw/cWFVf48+M9cSQneEiJ95Ic7yE5wUtKvIfGafGkJvzenxzvJSWh1Hu8d795UxK81EyM/NWjrhUVEakDfAS0ANYAA1V1exnTfQ2cDPyoqr1LDG8JjALqArOBK1W1MPLJw09ESE1NBYKXNxcVFdk5FGNCoKoU+Iv3+yLfXeAv/1d/WYXBGV6yCBQGKl4AEn1xZX6R16+RUGp48Is/JcGZpsS4kgUhKd5DvDf2Tnu7uadyNzBFVYeLyN1O/11lTPc0kAxcV2r4k8BzqjpKRF4DBgGvHm6ohz9fxOINOw93Mfs59sia/OOi4w44TSAQ4IQTTiAjI4MhQ4bQtWvXsGYwJlqoOsf/yznUs7uMX/cH+/WfXxggUKwVzrDvC7/kr/y0JB+N0xKdL/WD7wWULAzJ8V48cfZDENwtKn2AM53ud4HvKKOoqOoUETmz5DAJ/ow/G7i8xPwPEYai4haPx8PcuXPZsWMHffv2ZeHChbRv397tWMaEZPPOvbz3y1pWbd1dbtHILwqgFfz+jxP2/5XvfKnXS42neULyH8aV3EPY773E+CSfh7hIFABVKPZDoNB5FTmvwhLvhQefprho/+nDMo3TffMsSKgR/nUvwc2i0lBVs53ujUDDEOatC+xQVb/Tnwk0Dkeog+1RRFpaWhpnnXUWX3/9tRUVEzPW5eTz+g8r+WRWJv7iYlrWSyE1IXio58g0X6lf+c5eQDl7Ar8Vhvg4EiSA/OHLsdR7cX7ZX657imD3oX4Bl56ugsUgUsQDnnjw+JxX/O/vcSWHxYM3HhJSy55GIn84LaJFRUQmA0eUMeq+kj2qqiJS8X3X0HMMBgYDNGvWLFIfc8i2bNmCz+cjLS2NPXv2MGnSJO66q6wjgcZEl+WbdvHqdysZP28DHhGuPj6JG1OnkVaYXeoLuBB2VeRLusR4rdgVTYckzlfxL+n4ZPCkHWCaMpbhiYc47+/d5X5OfDnLcfr3fU6cJ3J/izCLaFFR1XPLGycim0Skkapmi0gjYHMIi84B0kTE6+ytNAGyDpBjBDACID09PWLF61BlZ2dz1VVXEQgEKC4uZuDAgfTu3fvgMxrjknnrd/Dy1Ay+WbyJ5HgPd3SBK4s/Y9H0r5i2PIm4wjgKEoWCJKEwSShMFIqSPPiT4vAneyhO8hJI8VKcmoTX58MnHrxxXnxxPnxxvmC3x4cvLh6v8+7zxOP1xOPzJODzJAS7vYn4vAl4PYlOdyJebyI+bzI+XyJebxI+XzJebyLiTSjxi93Of0SKm4e/xgNXAcOd93EVndHZs5kKXELwCrCQ5o82HTt2ZM6cOW7HMOaAVJXpq7bxyncZTFuxlVqJXp5K302vXZ8wc8r/+HlxEk02pSBJws5GNaiVW0TihiKS8v3EFQP4y1zuXh/sThJ2JSm7E4XdSbAjCXYn7hteVjcEPKEVhn1F6w/FK86Hz+PDK/sKWTnTlJ7+INP8Xhj/OM0f+stYnjfOS1wlHK4KNzeLynDgYxEZBKwFBgKISDpwvape4/RPA9oBqSKSCQxS1YkET+qPEpF/AnOAN11YB2OqPFXl26WbeXlqBr+u20HDVC8j07PonP0BM7/IZN7iBBrtTGZLPR8bb+zFKYPuJTGl5n7zF+flEdiRS2DHDgK5OyjOzSWQ6/TvCHb7nXGBHTsIbM6leOdO8Jd/CEwTEyiumUJxjWQCNZLwpybjT02kKDWBotQEClLiKUjxUZDiY0+Kjz1JHvYkeyiMK6aouAh/sZ+i4qL9uwNF+NVPgb+A3cW7fxte3rRFxUUokTv44RVvSIXvYNPe3Plmkn3JEcsLLhYVVc0Bzilj+CzgmhL9p5cz/yrgpIgFNKaaCxQrXy7I5pWpGSzduItWaXF81GURDTL+w4JP8li3xEfLwgQyj6qJ746/clq/64nz/PHYv4jgSU3Fk5oKTSp+PU1Fi1FwXC6BDbkEcjMJ5OaCv+y9IgBJTsaTVgtPrTQ8tWrhSUv7vXtff1qJbme4xJd9J3qgOHDQIrWvAJU5zb7uUtOEMu2+V74//4+fX2KaIZ2GVPjvf6jsjnpjzH4K/AHG/prFa9+vZE1OPl3qFTPh+F8oXPAJ6z8QUlZ4aIWP9Sc2odUNt9HjlF4RyVFZxahg+fJg/yEVo1r7FSNvWi0SKliMqiorKsYYAPIL/Xw4cz0jf1jFxp176XFEPm8eM5ms2d+w7qsEmmd5OTIB1vc6ns433k/71tF5yXtYipFzGK7MYuT0h7MYhbJnFO2sqBhTzeXuKeK9n9fw1k+r2Z5fxOWNtzC47niW/m8OmYsSabA9kW21PGT9/RxOvvY+TqjdwO3IEWHFKDysqBhTTW3ZVcBbP63m/V/WkldQyNBma+i9ZzSLp2exebGP5nsS2dA0iW3X/5mul92KNz7B7chRKSLFKDeXwPYdYS9G9a4bTFxKSjhWu1xWVKJIixYtqFGjBh6PB6/Xy6xZs9yOZKqgzO35jPxhFaP+tx4CBTzQbCEdN33Cmsl7yFvmoVXAx7qO9UkaPISzzxlojZtGiBvFqN71pZtQDD8rKlFm6tSp1KtXz+0YpgrK2Lyb175fyWdzsqgpeTzbeAZpq74g9zPFtzaOZl4PmWccTfsh93DBcfbYhWh1OMWoMlhRMaaKW5iVyyvfZfDVwo208G7jrUZTKVj0PZ4ffNTZInhT4lj351M56cYH6dSgqdtxTYRU1h6nFZXSvrobNi4I7zKP6AAXDD/oZCJCz549ERGuu+46Bg8eHN4cplr535pt/PvbDL5fvoUTEjIZVe8rts5bgG9CPPXz4tnUMJ5NQ/vQ7aq7iE+K7HF2U31YUYkiP/74I40bN2bz5s306NGDdu3a0b17d7djmRiiqny/fAuvTF3JzDU59EpeyqjU8Wyel03CUi+tiuJZ17YWvr9fQ/eL/k5cXOw1A2KimxWV0iqwRxEpjRsHj482aNCAvn37MnPmTCsqpkICxcrERRt5eWoGyzZs44rUWdwkn7Nzxh5qrIwjJc7L+q5NaXPjnZx3YrntvBpz2KyoRIm8vDyKi4upUaMGeXl5fPPNNzz44INuxzJRrihQzGdzsnj1+5Vs2rKVG1K/57b8yRTPUOpnC8mJcazr04UuNz5Ah+bt3I5rqgErKlFi06ZN9O3bFwC/38/ll1/O+eef73IqE632FgX4eNZ6Xv9+FYU7NnBr6tfU2TKTGlM91MuFrXW8ZA0+j27X3E9yzdpuxzXViBWVKNGqVSvmzZvndgwT5XbtLeKD6et488dV1MpbzR3x4/CsXc6Ri70kF3jIbJHMjqFX0m3AEDxen9txTTVkRcWYGLAtr5C3f1rNOz+v5piChTwkn6GLN9NihYc49bK2UwOa3zCUHt37uh3VVHNWVIyJYtm5exj5w2o+mrmG7oGfGZ7/BXGL9tBifRx74z2sP7stHW9+kF5tu7gd1RjAiooxUWnN1jxe+34lX/66kj46iSe3fkfqwmIabhN21PCw7orTOOWGh+lcr5HbUY3ZjxUVY6LIkuydvPrdSn6ev5QrGMdjmXNouCiOmvmQ3SiBLcMu4ZS/DMOXmOR2VGPKZEXFmCjw67rtvDI1gxVL5/NX/8ecu2o1zZd5iPfHseaYWvgG38CZ511pNyuaqGdFxRiXqCo/ZeTw8tQM8lf9wsA9H9Nv+TZaro6j0OMh65SmtLvpfi7oZDfAmthhRSVKLFu2jEsvvfS3/lWrVvHII48wdOhQF1OZSCguViYt2cSr3y6nbtZk+u0YR+3FBTTeLOxKjmNt386k3/woxx/Z2u2oxoTMikqUaNu2LXPnzgUgEAjQuHHj326GNFWDP1DM5/M3MPLbpbTfMoa/bvyOIxYptXfB5npesm+4gFOuuZ+klFpuRzXmkFlRiUJTpkyhdevWNG/e3O0oJgz2FgX49NdMPvhuHmds+Q+D18+j2RIhqRDWtUrCc9ffOL3vjcR5PG5HNeawWVEp5cmZT7J029KwLrNdnXbcddJdFZ5+1KhRXHbZZWHNYCpfXoGf/85Yx+c/TKfn1ne4ceVaWmXEUSzC+s71aDXkbs7rdqHbMY0JKysqUaawsJDx48fzxBNPuB3FHKId+YW88/Mafpk2ifM2v8+QZTtotkHIT4hjbY82dL7tcTq0bO92TGMiwopKKaHsUUTCV199RZcuXWjYsKGrOUzoNu/cyxvTVrH254/pnj2WIYsLqL9D2FYrjvVXnkq3IY9zQlp9t2MaE1FWVKLMhx9+aIe+Ysz6bfmM/H4Ze6eP5MR139FjiZK6F7Iax5Nz3QBOvuJOvPEJbsc0plJYUYkieXl5TJo0iddff93tKKYCVmzaxVtTFlBj1nN0WbWAVisETwDWHleD+OuGck6PyyrtueDGRAsrKlEkJSWFnJwct2OYg5i3fgcfTPqZpr8+w+nLs2i5TijwCeu7HUn7W/9Jr/anuB3RGNdYUTGmAlSV6au28dlXX3D03FfotTSXI7YKuSnC2n4d6Tr0aTo1aOZ2TGNcZ0XFmANQVaYu3cR3n79D2/kf0ndJIbXyYGMDL5tu6Em3wY8Sn5TidkxjooYVFWPKEChWvpq3nnljn6Lt4qn0W1ZMQhGsOSoR79/+zhl9h1jjjsaUwYqKMSUU+osZN3MZ68c8SJvFC7l4FQQ8sL5TbY6+5X4u6NrL7YjGRDUrKsYA+YV+xnz3P3Z+dj9tFm+g/UbYnQhrz2tJ+u3P0LH5MW5HNCYmWFEx1VruniLGfDEeJjxFm0U7qbsTttQWsv5yEqfc8iwpNeu4HdGYmGJFJUq88MILjBw5ElXl2muvtSbvI2zr7gLG/fdlkqa8S8clhSQXwrqmXuJu6MupVz6Ax+tzO6IxMcm1oiIidYCPgBbAGmCgqm4vY7qvgZOBH1W1d4nh7wBnALnOoKtVdW5kU0fGwoULGTlyJDNnziQ+Pp7zzz+f3r17c9RRR7kdrcrJ2p7H1yPuo/ZPkzgpoxhRWHVsMq2vv5XzevzV7XjGxDw3L1+5G5iiqm2AKU5/WZ4Grixn3B2q2sl5xWRBAViyZAldu3YlOTkZr9fLGWecwZgxY9yOVaWsyNzIm3cPZP6AdLq9PZEWa4pZdUp90ka/wZ8+nc1xVlCMCQs3D3/1Ac50ut8FvgP+0Jqjqk4RkTNLD4+UjY8/TsGS8DZ9n3BMO464995yx7dv35777ruPnJwckpKSmDBhAunp6WHNUF3NW7CAOa/dRstfs+i2HbbXgJUXHc3pd/ybLg2auh3PmCrHzaLSUFWzne6NwKE0y/uYiDyIs6ejqgVhS1eJjjnmGO666y569uxJSkoKnTp1wmMPbDos06Z+ydq3H+boBbvougeyGgqZ15zOGTc9S3yi3axoTKREtKiIyGTgiDJG3VeyR1VVRDTExd9DsBjFAyMI7uU8Uk6OwcBggGbNDtyUxoH2KCJp0KBBDBo0CIB7772XJk2auJIjlqkqEz96iV2fvMnRSws5IQArWvsI/OVyzr70TrtZ0ZhKENGioqrnljdORDaJSCNVzRaRRsDmEJe9by+nQETeBoYdYNoRBAsP6enpoRavSrF582YaNGjAunXrGDNmDNOnT3c7UszwF/n5/N/D8E6cxFFriin0QkaHFI4dci9/Or2f2/GMqVbcPPw1HrgKGO68jwtl5hIFSYCLgYXhj1h5+vfvT05ODj6fj5dffpm0tDS3I0W9osICxj58JXV/WEC7LbAzGRZ0b8jpdz7P8Ud1cjueMdWSm0VlOPCxiAwC1gIDAUQkHbheVa9x+qcB7YBUEckEBqnqROA/IlIfEGAucL0L6xA206ZNcztCzPn0urM5/pdtbKwDC/u0o9e9I+hay56saIybXCsqqpoDnFPG8FnANSX6Ty9n/rMjl85Eu4lv/oMO07ex+NgE+nw0E68v3u1IxhjcvU/FmEOycd1ykkd8zPYa0O25UVZQjIkiVlQcqlF5/v430Z6vMv1420Dq7ITd115Co+bt3I5jjCnBigqQmJhITk5O1H5xqyo5OTkkJia6HcV1n/7zao5bVMCCk2tz/rWPuh3HGFOKNSgJNGnShMzMTLZs2eJ2lHIlJiZW+3tXlvxvMs1GzyCzodD7ha/cjmOMKYMVFcDn89GyZUu3Y5gDKCosYOX9t9LED6l33UVqzVpuRzLGlMEOf5mYMPqOPrReW8zSHq04pddVbscxxpTDioqJej+PH8GxU9aS0dLDgKdDukfWGFPJ7PCXiWq7c7eR//RzeH3Q5vF/4/HaP1ljopntqZio9uXNvWi8BTIHnkq7zme6HccYcxBWVEzU+vLVe2g/M5dFHRLpd88bbscxxlSAFRUTlbJWLyLt7c/ISYPuz492O44xpoKsqJioNOP2y6m1C/ZefwUNGrd2O44xpoKsqJioM/qhKzhmSSELu9Wj59X3ux3HGBMCKyomqiycPoEWY39l3RFCnxe/dDuOMSZEVlRM1CgqLGDtA3fgKYba9z5AUkpNtyMZY0JU4aIiIk+JSE0R8YnIFBHZIiJ/iWQ4U718etuFtFpfzPLzj+aknpe5HccYcwhC2VPpqao7gd7AGuAo4I5IhDLVzw+j/82x32WxvLWXS5741O04xphDFEpR2Xcr84XAJ6qaG4E8phrakZON//mXyU+A9sNft7vmjYlhoRSVL0RkKXACMMV5PvzeyMQy1cnEW/5Eo62w6fKzad2hm9txjDGHocJFRVXvBroB6apaBOQBfSIVzFQPX7z0f3ScvZsFxyfTZ9jLbscxxhymUE7UDwCKVDUgIvcDHwBHRiyZqfLWLfuVOu9OYFMdOPfFz9yOY4wJg1AOfz2gqrtE5DTgXOBN4NXIxDJVXcDvZ/YdV1MjH4pvGkSdhk3djmSMCYNQikrAeb8QGKGqXwLx4Y9kqoNP/3E57ZYXsej0hpx9+TC34xhjwiSUopIlIq8DlwITRCQhxPmNAWDO92Np/fkC1jYW+j4/we04xpgwCqUoDAQmAuep6g6gDnafiglRwZ58Nj5yHwI0fOCfJCQlux3JGBNGoVz9lQ+sBM4TkZuABqr6TcSSmSppzG0X0iJLyeh9HJ3P7Od2HGNMmIVy9detwH+ABs7rAxG5OVLBTNXz7YfP0P6HjSxt46X/I6PcjmOMiYBQbl0eBHRV1TwAEXkS+AV4KRLBTNWybdN64l56g93J0Pmpt+2ueWOqqFDOqQi/XwGG0y3hjWOqqsm39qXhNth65fm0OCbd7TjGmAgJ5efi28AMERnr9F9M8F4VYw5o3DM30WFuHgtOSGXgrc+5HccYE0EVLiqq+qyIfAec5gz6m6rOiUgqU2WsWjyThv+dwsa60PPF8W7HMcZE2EGLiojUKdG7xnn9Nk5Vt4U/lqkKAn4/8+8cRKu9sPfuG0mr28jtSMaYCKvInspsQPn9/Ik67+J0t4pALlMFjL5/AB0z/Mw7+0j+PMAuFDSmOjhoUVHVlhVZkIgcp6qLDj+SqQpmTRlFmwlLWd00jn5217wx1UY4m1l5P4zLMjFsT95Ocv75CCrQ5OGniI9PcDuSMaaShLOohHR5sYjUEZFJIrLCea9dxjSdROQXEVkkIvNF5NIS41qKyAwRyRCRj0TEGreMEuNuvZBm2cqqPp3o2O1Ct+MYYypROIuKHnyS/dwNTFHVNsAUp7+0fOCvqnoccD7wvIikOeOeBJ5T1aOA7QRvzjQum/ze47T/aStL2sUz4JEP3Y5jjKlkbrYy3Ad41+l+l+B9L/tR1eWqusLp3gBsBuqLiABnA6MPNL+pXFuzV+N79X1yU+HEf/3H7TjGGBeEs6gUhjh9Q1XNdro3Ag0PNLGInETw+S0rgbrADlX1O6MzgcYhfr4Js6m39qfedtjxtz/R9Kj2bscxxrigwjc/ikiXMgbnAmtV1a+qJ5cxz2TgiDLmu69kj6qqiJR7+ExEGhG8EOAqVS0O7qhUnIgMBgYDNGvWLKR5TcWMffI62s/fw/wTa3LpjU+6HccY45JQmml5BegCzCd4Ur49sAioJSI3lNUMvqqeW97CRGSTiDRS1WynaGwuZ7qawJfAfao63RmcA6SJiNfZW2kCZJX3Wao6AhgBkJ6eHuq5H3MQy+b+wJEf/cCG+nDBi5+7HccY46JQDn9tADqrarqqngB0BlYBPYCnDuGzxwNXOd1XAeNKT+Bc0TUWeE9V950/QVUVmApccqD5TeQF/H6W3XMjiQWQMOw2atZu4HYkY4yLQikqR5e8uVFVFwPtVHXVIX72cKCHiKwAznX6EZF0EXnDmWYg0B24WkTmOq9Ozri7gNtFJIPgORZr3NIFo++6mDarAyw5uymn9RnsdhxjjMtCOfy1SEReBfY9XelSYLHzrPqiUD9YVXOAc8oYPgu4xun+APignPlXASeF+rkmfGZ89S5tv1nJymZx9H/GDnsZY0LbU7kayACGOq9VzrAi4KxwBzPRLW9XLrnDn6TIA60eewGf3TVvjCG0pu/3iMhLwDcEb3Rcpqr79lB2RyKciV6f39KL4zcpiy8/kf4nlns9hjGmmgnlkuIzCd5kuIbg1V9NReQqVf0hMtFMtJr45kN0mL6Nxccm0P/B99yOY4yJIqGcU3kG6KmqywBE5GjgQ+CESAQz0WnjuhUkj/iI7TWg23OjDj6DMaZaCeWcim9fQYFgEyqAL/yRTDT78bYB1NkJu6+9hEbN27kdxxgTZULZU5nlXOq772qsK4BZ4Y9kotWnj13NcYsKmHdybf587aNuxzHGRKFQisoNwBDgFqd/GsG77E01sHT2FJqOnkFmQ6H3C1+6HccYE6VCufqrAHjWeZlqJOAvIuPeW2hSBCl33kFqrT88+sYYY4AKFBURWcABnpWiqh3DmshEnU+GXcTxa4uZd0Er/nzh39yOY4yJYhXZU+kd8RQmav08fiTHTF5LRksPA5625tWMMQd20KKiqmsrsiAR+UVVTzn8SCZa7M7dRv7Tz+L1wdGP/xuPN5RTcMaY6iicD+lKDOOyTBT48pYLabwFMgeeStvOZ7odxxgTA9x8Rr2JYhNeu5f2M3awqEMi/e554+AzGGMM7j6j3kSprNWLqfXWWHLSoPvzow8+gzHGOMJZVEJ7xq+JWjNuv4xau2Dv9VfQoHFrt+MYY2JIOIvKlWFclnHJ6Ieu4JglhSzsVo+eV9/vdhxjTIwJpZXiXfzxvEkuwaZa/k9VF4YzmKl8C6d/RYuxv7LuCOFPL9pd88aY0IVyjejzQCbwX4KHuv4MtAZ+Bd4Czgx3OFN5igoLWPvAMI4shtr3PkBySk23IxljYlAoh7/+pKqvq+ouVd2pqiOA81T1I8Da7Yhxo2+/kFbri1l+/tGc1PMyt+MYY2JUKEUlX0QGikic8xoI7HXG2eXEMWzamJc5bmoWy1t7uOSJT92OY4yJYaEUlSsInozfDGxyuv8iIknATRHIZirBjpxsip79N/kJ0H74CLtr3hhzWEJppXgVcFE5o38MTxxT2Sbe0oeOW2HZNWdxcodubscxxsS4Cu+piMjRIjJFRBY6/R1FxK45jWFfvPR/dJy9i4XHJ3PxMHs0jjHm8IVy+GskcA9QBKCq8wleAWZi0Prl86jz7gQ21YFzXvzM7TjGmCoilKKSrKozSw3zhzOMqRwBv59Zw66kRj4U3zSIOg2buh3JGFNFhFJUtopIa5wrvUTkEiA7IqlMRH36j8tpt7yIhac14OzLh7kdxxhThYRyqc8QYATQTkSygNUErwgzMWTutHG0/nwBa48U+r3wldtxjDFVTChFJQt4G5gK1AF2AlcBj0Qgl4mAgj35ZD90Dw2Bhg88SkJSstuRjDFVTCiHv8YRvKS4CNgA7AbyIhHKRMaY2y6kRZay4sLj6HxWf7fjGGOqoFD2VJqo6vkRS2IiauqoZ2n/w0aWtfFyyaOj3I5jjKmiQtlT+VlEOkQsiYmY7Vuy4MWR7E6GTk+9bXfNG2MiJpSichowW0SWich8EVkgIvMjFcyEz6Rb+nDENth65Xm0OCbd7TjGmCoslJ+sF0QshYmYcc/cTIc5eSzoksrAW593O44xpooLpe2vtZEMYsJv1eKZNPzvZDbWhR4v2F3zxpjIC+fjhE0UCfj9zL9zEMl7QYbeSO36jd2OZIypBlwrKiJSR0QmicgK5/0PD/oSkU4i8ouILHLO41xaYtw7IrJaROY6r06VuwbRbfT9A2ib4WfxGUdy5oCb3Y5jjKkm3NxTuRuYoqptgClOf2n5wF9V9TjgfOB5EUkrMf4OVe3kvOZGPnJs+HV5RPEcAAARjElEQVTKJ7SZsJTVTeLo9/wEt+MYY6oRN4tKH+Bdp/td4OLSE6jqclVd4XRvIPiAsPqVljAG7cnbyebH/oEKNHl4OPHxCW5HMsZUI24WlYaquq9Byo1AwwNNLCInAfHAyhKDH3MOiz0nIvbtCYwb2pvmG5RVfTrR8dTynqlmjDGREdG74ERkMnBEGaPuK9mjqioi5T7nXkQaAe8DV6lqsTP4HoLFKJ5gQ5d3UU47ZCIyGBgM0KxZsxDXInZMfu9x2v+4hSVtfQx45EO34xhjqqGIFhVVPbe8cSKySUQaqWq2UzQ2lzNdTeBL4D5VnV5i2fv2cgpE5G2g3DbcVXUEwcJDenp6ucUrlm3NXo3v1ffJTYX0f33gdhxjTDXl5uGv8QRbOcZ5H1d6AhGJB8YC76nq6FLjGjnvQvB8zMKIpo1yU4f2p9522H51b5q16eh2HGNMNeVmURkO9BCRFcC5Tj8iki4ibzjTDAS6A1eXcenwf0RkAbAAqAf8s3LjR4+xT11P+3l7WHhiTXoPedrtOMaYakxUq+TRoHKlp6frrFmz3I4RNsvm/sCOv1/H7hQ4cfz31KzdwO1IxpgqSERmq+pBGw+0O+pjWMDvZ9k9N5JYAPG332oFxRjjOisqMWz03X1pszrAkrObcnrf692OY4wxVlRi1Yyv36PtxAxWNouj/zOfux3HGGMAKyoxKW9XLrlPDMfvgZaPPo/P7po3xkQJKyox6PNbetF0k7K234kc17WH23GMMeY3VlRizMS3HqHD9G0sPjaB/g++53YcY4zZjxWVGLJx3QqSX/+Q7TWg23Oj3I5jjDF/YEUlhvx42wDq7IRd11xCo+bt3I5jjDF/YEUlRnz62NUct6iABV3TuGDwo27HMcaYMllRiQFLZ39L09EzyGwo9H7hS7fjGGNMuayoRLmA30/GvTcTXwQpd95Baq06bkcyxphyWVGJcp8M603rtcUs7dGSbhf+ze04xhhzQFZUotgvX7zBMZPXktEijgFPj3c7jjHGHFREH9JlDt3u3G3kPfUMHh8c/cTLeLy2qYwx0c/2VKLUF7dcSOPNkDnwVNp2PtPtOMYYUyFWVKLQhNfupcOMHSxqn0i/e944+AzGGBMlrKhEmazVi6n51lhy0qD7C6MPPoMxxkQRKypRZvrtl5G2C/ZefzkNGrd2O44xxoTEikoUGf3QFRy7pJCF3erS8+oH3I5jjDEhs6ISJRZO/4oWY39l3RHCRXbXvDEmRllRiQJFhQWsfXAYnmKofe+9pKTWcjuSMcYcEisqUeDT23vTal0xy84/mpN6/sXtOMYYc8isqLhs2piXOXZqJstbeRjwxKduxzHGmMNiRcVFO3KyKXr23+QnQPsnR9hd88aYmGdFxUUTb+1Do62w8fKzaN2hm9txjDHmsFlRcckXL/0fHWftYsHxyVw87BW34xhjTFhYUXHBuhXzqPPuBDbVgXNf/MztOMYYEzZWVCpZwO9n9rArqZEPgSF/o07Dpm5HMsaYsLGiUsnGPHQF7ZYVsfC0BpxzxZ1uxzHGmLCyolKJ5k4bR6vx81l7pNDvha/cjmOMMWFnRaWSFOzJJ/uhexCg4QOPkpCU7HYkY4wJOysqlWTs7b1pkaVkXHgsnc/q73YcY4yJCCsqlWDqqGc57vtslrXx0v/Rj9yOY4wxEWNFJcK2b8mCF0eyOwk6PfW23TVvjKnSrKhE2KRb+nDENth85Xm0OCbd7TjGGBNRrhUVEakjIpNEZIXzXruMaZqLyK8iMldEFonI9SXGnSAiC0QkQ0ReFBGp3DU4uPHP3UKHOXks6JJKn6HPux3HGGMizs09lbuBKaraBpji9JeWDZyiqp2ArsDdInKkM+5V4FqgjfM6P/KRK27V4pk0+GASG+tCjxfsrnljTPXgZlHpA7zrdL8LXFx6AlUtVNUCpzcBJ6+INAJqqup0VVXgvbLmd0vA72f+nYNI2QMy9EZq12/sdiRjjKkUbhaVhqqa7XRvBBqWNZGINBWR+cB64ElV3QA0BjJLTJbpDIsKo+8fQNsMP4vObMSZA252O44xxlSaiF6KJCKTgSPKGHVfyR5VVRHRspahquuBjs5hr89EZPQh5BgMDAZo1qxZqLOHZPaUT2gzYSmrm8TR95kvIvpZxhgTbSJaVFT13PLGicgmEWmkqtnO4azNB1nWBhFZCJwO/AQ0KTG6CZB1gHlHACMA0tPTyyxe4bAnbydbHvsH9QWaPDzc7po3xlQ7bh7+Gg9c5XRfBYwrPYGINBGRJKe7NnAasMw5bLZTRE52rvr6a1nzV7ZxQ3vTfIOy8k/H0/HUi9yOY4wxlc7NojIc6CEiK4BznX5EJF1E3nCmOQaYISLzgO+Bf6nqAmfcjcAbQAawEnC1hcbJ7w+n/Y9bWNLWx8BHR7kZxRhjXCPBi6eqj/T0dJ01a1ZYl7k1ezWL+vXC64em/x1FszbHh3X5xhjjNhGZraoHvYPb7qgPg6lD+1NvO2y/urcVFGNMtWZF5TCNfep62s/bw8L0mvQe8rTbcYwxxlVWVA7Dink/ceSo79lQHy546XO34xhjjOusqByigN/PknuuI7EA4m+/hZq1G7gdyRhjXGdF5RB9cndf2qwKsOTsppze9wa34xhjTFSwonIIZnz9Hu0mZrCqWRz9n7HDXsYYs48VlRDl7cold/hw/B5o8ehz+OIT3I5kjDFRw4pKiD6/tRdNNypr+53IcV17uh3HGGOiihWVEEx86xE6/LKNxccm0P/B99yOY4wxUceKSgVtXLeC5Nc/ZHsN6PacNcNijDFlsaJSQT/ePpA6O2HXNf1o1Lyd23GMMSYqWVGpgII9+QAs6JrGBYMfczmNMcZEr4g+T6WqSEhK5pLRcwj4/W5HMcaYqGZ7KiHweK0GG2PMgVhRMcYYEzZWVIwxxoSNFRVjjDFhY0XFGGNM2FhRMcYYEzZWVIwxxoSNFRVjjDFhI6rqdoZKJSJbgLWHOHs9YGsY47ipqqxLVVkPsHWJVlVlXQ53PZqrav2DTVTtisrhEJFZqprudo5wqCrrUlXWA2xdolVVWZfKWg87/GWMMSZsrKgYY4wJGysqoRnhdoAwqirrUlXWA2xdolVVWZdKWQ87p2KMMSZsbE/FGGNM2FhRKYeIrBGRBSIyV0RmOcPqiMgkEVnhvNd2O2dFlLMuD4lIljNsroj0cjtnRYhImoiMFpGlIrJERE6J4e1S1rrE1HYRkbYlss4VkZ0iMjQWt8kB1iWmtsk+InKbiCwSkYUi8qGIJIpISxGZISIZIvKRiMSH/XPt8FfZRGQNkK6qW0sMewrYpqrDReRuoLaq3uVWxooqZ10eAnar6r/cynUoRORdYJqqvuH8h0gG7iU2t0tZ6zKUGNwuACLiAbKArsAQYnCb7FNqXf5GjG0TEWkM/Agcq6p7RORjYALQCxijqqNE5DVgnqq+Gs7Ptj2V0PQB3nW63wUudjFLtSMitYDuwJsAqlqoqjuIwe1ygHWJZecAK1V1LTG4TUopuS6xygskiYiX4A+WbOBsYLQzPiLbxYpK+RT4RkRmi8hgZ1hDVc12ujcCDd2JFrKy1gXgJhGZLyJvxcLhCaAlsAV4W0TmiMgbIpJCbG6X8tYFYm+77PNn4EOnOxa3SUkl1wVibJuoahbwL2AdwWKSC8wGdqjqvueiZwKNw/3ZVlTKd5qqdgEuAIaISPeSIzV43DBWjh2WtS6vAq2BTgT/0T3jYr6K8gJdgFdVtTOQB9xdcoIY2i7lrUssbhecw3d/Aj4pPS6GtglQ5rrE3DZxCl8fgj9ejgRSgPMr47OtqJTDqfSo6mZgLHASsElEGgE475vdS1hxZa2Lqm5S1YCqFgMjCa5ftMsEMlV1htM/muAXcyxulzLXJUa3CwR/sPyqqpuc/ljcJvvsty4xuk3OBVar6hZVLQLGAKcCac7hMIAmBM8bhZUVlTKISIqI1NjXDfQEFgLjgaucya4CxrmTsOLKW5d9/+EdfQmuX1RT1Y3AehFp6ww6B1hMDG6X8tYlFreL4zL2P1wUc9ukhP3WJUa3yTrgZBFJFhHh9/8rU4FLnGkisl3s6q8yiEgrgr/oIXiY4r+q+piI1AU+BpoRbOl4oKpucylmhRxgXd4nuDuvwBrguhLHwKOWiHQC3gDigVUEr8yJI8a2C5S7Li8SY9vF+bGyDmilqrnOsJj7vwLlrkus/l95GLgU8ANzgGsInkMZBdRxhv1FVQvC+rlWVIwxxoSLHf4yxhgTNlZUjDHGhI0VFWOMMWFjRcUYY0zYWFExxhgTNlZUjDHGhI0VFVMticg7InLJwaesHCLSRkRmOu1LTXY7T1lEZPdhzDtURJLDsSwT3ayoGBMd7ibYDlhH4Fq3w0TAUIIt5ZoqzoqKiQoi0sJ5UNVI58FC34hIkjPuOxFJd7rrOc+HQUSuFpHPnIdArRGRm0TkdqfV3+kiUqeCn32CiHzvtOI8sUSbVdeKyP9EZJ6IfOo0eVFLRNaKSJwzTYqIrBcRn4i0FpGvneVME5F2zjQDJPigpHki8kM5MQoJtsWEqq4+QNYznazjRGSViAwXkSucvZwFItLame4iCT6MaY6ITBaRhs7wF0TkQaf7PBH5Yd+6lPFZLUXkF2e5/yw17g7nbzPfuXN73zZcKiL/cbblaOdvdgvBRg2nisjUEst4zPmbTN+Xz1QBqmove7n+AloQbE6ik9P/McEmJAC+I/iQMYB6wBqn+2ogA6gB1CfYvPf1zrjngKEH+Lx3CLaB5AN+Buo7wy8F3nK665aY/p/AzU73OOCsEtO/4XRPAdo43V2Bb53uBUBjpzutnDzDCDaF3/sgf6czgR1AIyCBYIOADzvjbgWed7pr83uLGdcAzzjdycAi4CxgGdD6AJ81Hvir0z2E4IOqINh+3AhACP4w/YLgs2FaEGzK5FRnureAYU73GqBeiWUrcJHT/RRwv9v/Bu0Vnte+1iqNiQarVXWu0z2b4JfUwUxV1V3ALhHJBT53hi8AOlZg/rZAe2BSsN09PASbNwdo7/xCTwNSgYnO8I8IFpOpBJ+78YqIpALdgE+c5UDwSx/gJ+AdCT59b0zpACLSheAXdWcnxzbgF2AlwS/90m0p/U+dtqdEZCXwTYl1PsvpbgJ85Ox1xQOrAVQ1X0SuBX4AblPVlQf425wK9He63weedLp7Oq85Tn8q0IZgm1nrVfUnZ/gHwC0En+tRWiHBYgTBbd3jADlMDLGiYqJJyYbtAkCS0+3n90O1iQeYp7hEfzEV+/ctwCJVPaWMce8AF6vqPBG5muBeAgR/wT/uHF47AfiW4PMqdqhqp9ILUdXrRaQrcCEwW0ROUNWcEpOcC/ysqpki0tdZ/mvAhDIKSkXX+SXgWVUdLyJnAg+VmKcDkEPwkNTBlPX5Ajyhqq/vN1CkRRnTl9e4YFGJdQtg30VVhp1TMbFgDcEvb/i92e5wWQbUF5FTAJxzI8c542oA2SLiA67YN4Oq7gb+B7wAfKHBZ23sBFaLyABnOSIixzvdrVV1hqo+SPAQV9NSGeYAfUSklqouBZ4m+CCoDw5jvWrx+7My9jVBj4g0B/6P4F7RBU6xK89PBPfEoMT6E9xj+7uzd4aINBaRBs64Zvv+lsDlBJ+TDrCL4N/TVHFWVEws+Bdwg4jMIXhOJWxUtZBgoXpSROYBcwkexgJ4AJhB8Mt1aalZPwL+4rzvcwUwyFnOIoJP3gN42jnZvZDg+Zt5pTJMIlhApovIbOA8gs3gvyMi9Q9x1R4ieChuNrAVgoUOeJPgeY4NwCDgDREpvfe3z60EnxS6gBKPnVXVb4D/Ar8440bze8FY5syzhOB5nVed4SOAr0ueqDdVkzV9b4wJC+fw1xeq2t7lKMZFtqdijDEmbGxPxVRpIvIywauYSnpBVd92I080EpH7gAGlBn+iqo+5kcfENisqxhhjwsYOfxljjAkbKyrGGGPCxoqKMcaYsLGiYowxJmysqBhjjAmb/wch6Nl8/SVZJwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_scores =  np.array(test_means).reshape(len(num_leaves_s),len(max_depth_s))\n",
    "\n",
    "for i, value in enumerate(max_depth_s):\n",
    "    plt.plot(num_leaves_s, test_scores[:,i], label= max_depth_s[i])\n",
    "    \n",
    "plt.legend(loc='best')\n",
    "plt.xlabel( 'num_leaves & max_depth' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本次搜索最优的是max_depth=9, num_leaves=50，但从图上发现num_leaves=70，max_depth=3才是最优的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3)min_data_in_leaf\n",
    "min_child_samples：叶子节点中最小的数据量，调大可以防止过拟合。\n",
    "搜索范围10-50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   45.9s\n",
      "[Parallel(n_jobs=5)]: Done  16 out of  20 | elapsed:  1.5min remaining:   22.5s\n",
      "[Parallel(n_jobs=5)]: Done  20 out of  20 | elapsed:  1.5min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=628,\n",
       "        n_jobs=5, num_leaves=70, objective='binary', random_state=None,\n",
       "        reg_alpha=0.0, reg_lambda=0.0, silent=False, subsample=1.0,\n",
       "        subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'min_data_in_leaf': range(10, 50, 10)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'num_leaves':70\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "min_child_samples_s = range(10,50,10) \n",
    "tuned_parameters = dict( min_data_in_leaf = min_child_samples_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5,  param_grid=tuned_parameters, cv = kfold, scoring='neg_log_loss', verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.3184025250485989\n",
      "{'min_data_in_leaf': 20}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAELCAYAAABwLzlKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xd4FOXax/HvnULoJRBq6L1DEgERERVBUURFUQREj6+IqFiPYjm2c7AdBEUEQY8NEHuBI0dARFBQNAkkdBAEEmroPUByv3/sRNe4pJBsZje5P9e1V2Znn5n5TTbJnZmdeR5RVYwxxhg3hLgdwBhjTMllRcgYY4xrrAgZY4xxjRUhY4wxrrEiZIwxxjVWhIwxxrjGipAxxhjXWBEyxhjjGitCxhhjXBPmdoBAV61aNW3QoIHbMYwxJqgkJCTsUdWo3NpZEcpFgwYNiI+PdzuGMcYEFRHZkpd2djrOGGOMa6wIGWOMcY0VIWOMMa6xImSMMcY1rhUhEYkUkXkissH5WsVHm/oikigiy0VklYgM93pttIikiMiRbMvUE5EFIrJMRJJFpI+P14+IyIP+2ztjjDF54eaR0Chgvqo2BeY7z7PbAZyrqh2AzsAoEantvDYL6ORjmceBj1S1I3ADMDHb62OB/xVCfmOMMQXkZhHqB7zrTL8LXJW9gaqeVNV052kEXnlV9SdV3eFjvQpUdKYrAduzXhCRq4DfgFUFTm+MMabA3CxCNbyKyE6ghq9GIlJXRJKBFOAFVd3uq52Xp4DBIpIKzAbudtZTHngYeLoQspsg9evuI6zZccjtGMYYh1+LkIh8IyIrfTz6ebdTVcVzBPMXqpqiqu2AJsBQEfFZrLwMBN5R1WigDzBVRELwFKdxqnokp4Wd3MNEJF5E4tPS0nLfURMUNu85yjUTF3PtpCX8ujvXHwNjTBEQz99/FzYssg7ooao7RKQW8J2qNs9lmbeA2ar6ide8I6pa3uv5KuBSVU1xnm8CugCfAnWdZpWBTOAJVZ2Q0zbj4uLUekwIfkfST3P1a4tJO5JOiAhVy5XiizvPo1yEdRpijD+ISIKqxuXWzs3TcTOBoc70UODL7A1EJFpEyjjTVYBuwLpc1rsVuNhZpiVQGkhT1fNVtYGqNgBeBp7NrQCZ4iEzU7n/w+Vs2nOU126M4dWBHdmYdoRHPluBW/+EGWM83CxCzwOXiMgGoKfzHBGJE5E3nTYtgaUikgQsBMao6gqn3YvO5z5lRSRVRJ5ylnkAuM1ZZgZws9pfmhJt/LcbmLt6F4/1acl5TapxXpNqPNCrOTOTtvPej3nq3soY4yeunY4LFnY6Lrh9vXInw6cl0D8mmjHXtUNEAM/R0bCp8Sxcn8YHw84ltv5fblMzxhRAMJyOM8av1u86zAMfLad93cqMvrrN7wUIICREeOm6DtSqVIY7pyey50h6DmsyxviLFSFTLB04dpLb3ounbEQYkwfHUjo89C9tKpUNZ9LgGPYfO8nIGcvIyLSzAsYUNStCptg5nZHJ3TOWsf3AcV4fHEPNSqXP2LZ17Ur886o2LNm4l7HzcrvmxRhT2KwImWLnxTnr+H7DHv7Zrw2x9SNzbT8gri4DO9XltQUbmbd6VxEkNMZksSJkipUvlm1jyqJNDOlSnxs61cvzck/2bU2bOhW5/6PlbNl71I8JjTHerAiZYmNF6kEe/jSZTg0jeaJvq3wtWzo8lEmDYgkRYfi0RE6cyvBTSmOMNytCplhIO5zOsKnxVCsfwcRBMYSH5v9Hu25kWV6+vgNrdx7i8S9W2o2sxhQBK0Im6J08ncmI6QnsP3aSyUNiqVY+4qzXdWGL6tx9UVM+SUjlg19SCjGlMcYXK0Im6D09axW/bN7PC/3b0aZOpQKv756Lm3J+02o8+eUqklMPFEJCY8yZWBEyQe39pVuZvnQrt1/QiH4d6hTKOkNDhFdu6Ei18qW4Y1oiB46dLJT1GmP+yoqQCVrxm/fx5MyVXNAsiod6tyjUdUeWK8XEwbGkHU7n3g+Xk2k3shrjF1aETFDacfA4w6clUqdyGcbf0JHQEMl9oXzqULcyT/RtxXfr0piw4NdCX78xxoqQCUInTmVw+9QEjp88zRs3xVGpbLjftjWocz2u6ViHcd+sZ+F6G+DQmMJmRcgEFVXlkc9WkJx6kHHXd6BpjQp+3Z6IMPrqtjSvUYF7PlhG6v5jft2eMSWNFSETVP7zw298vmwb91/SjF6taxbJNsuUCmXS4FgyMpQ7pyeSftpuZDWmsFgRMkHj+w1pPDt7DZe2rsldFzYp0m03rFaOf1/XnqTUg/zzv6uLdNvGFGdWhExQ2LL3KHe9v4ym1Svw0oD2hPjhQoTcXNqmJrdf0IhpP23ls8TUIt++McWRFSET8I6mn2bYewkATLkplnIRYa5l+Xuv5nRuGMmjn69gzY5DruUwpriwImQCWmam8sBHSWzYfZjXboyhftVyruYJCw3h1Rs7UrF0OHdMS+DQiVOu5jEm2FkRMgFtwoJf+XrVTh7t05JuTau5HQeA6hVK89qgGFL3H+fBj5Kso1NjCsCKkAlY81bvYuy89VzdsQ63dmvodpw/OadBJI/0acnc1buYvGiT23GMCVpWhExA2rDrMPd9uJx20ZV47pq2iBT9hQi5+dt5Dbi8bS1e/HotP27c63YcY4KSFSETcA4eO8Vt78VTOjyUyUNiKR0e6nYkn0SEF65tR4Nq5bh7RiK7Dp1wO5IxQceKkAkoGZnKyA+Wse3AcV4fHEOtSmXcjpSj8hFhTB4cy7GTGdw5PZFTGZluRzImqFgRMgHlxTlrWbg+jaevbENcg0i34+RJ0xoVeL5/O+K37Of5/611O44xQcWKkAkYXy7fxuSFmxjUuR43dq7ndpx8ubJ9bW7u2oD//PAb/03e7nYcY4KGFSETEFZuO8jDnyZzToMqPNm3tdtxzsqjfVoSU68yD3+SzK+7D7sdx5igYEXIuG7PkXRun5pAlbKlmDgollJhwfljWSoshNcGxVA6PJTh0xI5mn7a7UjGBLzg/G03xcapjExGTE9kz5F0pgyJI6pChNuRCqRWpTK8OrAjm9KOMOqzFXYjqzG5sCJkXPXMrNX8/Ns+XujfjrbRldyOUyi6NqnGg72bMytpO+8s2ex2HGMCmhUh45oPft7K1J+2MKx7I67qWMftOIVqePfG9GxZg9FfrSFhyz634xgTsKwIGVckbNnHP75cyflNq/HwpS3cjlPoQkKElwa0p3blMr+fbjTG/JVrRUhEIkVknohscL5W8dGmvogkishyEVklIsO9XhstIikiciTbMvVEZIGILBORZBHp4/VaOxH50VnXChEp7d+9NL7sPHiC4dMSqV25DBMGxhDqwthARaFSmXAmDY7hwLFT3P3+Mk7bjazG/IWbR0KjgPmq2hSY7zzPbgdwrqp2ADoDo0SktvPaLKCTj2UeBz5S1Y7ADcBEABEJA6YBw1W1NdADsH74i9iJUxncPjWeY+mneeOmOCqVDXc7kl+1rl2J0Ve35cdNe3lp3nq34xgTcNwsQv2Ad53pd4GrsjdQ1ZOqmnUeIwKvvKr6k6ru8LFeBSo605WArDsHewHJqprkLL9XVTMKvBcmz1SVxz5fSVLqQcZe34FmNSq4HalIXBsbzcBO9Zj03UbmrtrpdhxjAoqbRaiGVxHZCdTw1UhE6opIMpACvKCqud2O/hQwWERSgdnA3c78ZoCKyBznFN9DBd4Dky9vL97Mp4mp3HNxU3q3rul2nCL1ZN9WtK1TiQc+TmLznqNuxzEmYPi1CInINyKy0sejn3c79dxM4fOGClVNUdV2QBNgqIj4LFZeBgLvqGo00AeYKiIhQBjQDRjkfL1aRC4+Q+5hIhIvIvFpaWn52WVzBot/3cPo2Wvo1aoG91zc1O04Ra50eCgTB8UQIsLwaQkcP2kH4caAn4uQqvZU1TY+Hl8Cu0SkFoDzdXcu69oOrATOz2WztwIfOcv8CJQGqgGpwCJV3aOqx/AcJcWcYVtTVDVOVeOioqLyvsPGp617j3Hn+4k0qlaOsdd3IKSYXoiQm7qRZXn5hg6s23WYx79YaTeyGoO7p+NmAkOd6aHAl9kbiEi0iJRxpqvgOYJZl8t6twIXO8u0xFOE0oA5QFsRKetcpHABsLoQ9sPk4Gj6aYZNjSczU3njpjjKR4S5HclVFzavzsiLmvJpYiozfk5xO44xrnOzCD0PXCIiG4CeznNEJE5E3nTatASWikgSsBAYo6ornHYvOp/7lBWRVBF5ylnmAeA2Z5kZwM3qsR8YC/wCLAcSVfWrItnTEkpV+fsnSazfdZgJN8bQoFo5tyMFhJEXN6V7syiemrmK5NQDbscxxlVipwRyFhcXp/Hx8W7HCEoTvt3AmLnrebRPC4Z1b+x2nICy7+hJ+r76AwD/vbsbVcqVcjmRMYVLRBJUNS63dtZjgvGL+Wt28dK89VzVoTa3nd/I7TgBJ7JcKSYOiiHtcDr3friczEz7Z9CUTFaETKH7dfcR7vlgOa1rV+T5/u0QKZkXIuSmfd3KPHllKxauT2P8txvcjmOMK6wImUJ18Pgphr0XT0RYCJOHxFE6PNTtSAHtxk71uCamDq/M38B363K8QNSYYsmKkCk0GZnKvR8sY+u+Y0waHEudymXcjhTwRITRV7WleY0K3PvhclL3H3M7kjFFyoqQKTRj5q5jwbo0nryyNZ0aRrodJ2iUKRXKpMGxZGQoI6Ynkn7abmQ1JYcVIVMoZiVtZ9J3GxnYqR6DO9dzO07QaVitHC8NaE9y6kGenmW3r5mSw4qQKbBV2w/y90+SiKtfhaevbG0XIpylXq1rMvyCxry/dCufJKS6HceYImFFyBTI3iPpDHsvgcplSjFxcAylwuxHqiAe7NWMLo0ieezzFazefsjtOMb4nf3FMGftVEYmd76fSNqRdKbcFEv1CjZGYEGFhYbw6sAYKpUJ547pCRw8bkNemeLNipA5a6O/WsNPm/bx/DVtaRdd2e04xUZUhQgmDoph2/7jPPhxkt3Iaoo1K0LmrHz0SwrvLNnMrd0ack1MtNtxip24BpE82qcl81bvYvKiTW7HMcZvrAiZfEvcup/Hv1hJtybVeOSyFm7HKbZuOa8Bl7erxb/nrGXJxj1uxzHGL6wImXzZdegEw6cmUKNSBK8O7EhYqP0I+YuI8EL/djSsVo6RM5ax8+AJtyMZU+jsL4jJsxOnMrh9agJH0k/zxk1x1vNzESgfEcbkIbEcO5nBne8ncioj0+1IxhQqK0ImT1SVf3yxkuUpBxg7oD0talZ0O1KJ0aR6BV7o346ELft5dvYat+MYU6isCJk8eXfJZj5OSGXkRU24tE0tt+OUOH3b1+aW8xrw9uLNzEra7nYcYwqNFSGTqyUb9/DPr9bQs2UN7u3ZzO04JdYjl7Uktn4VHv40mV93H3Y7jjGFwoqQyVHKvmPcOT2RhtXKMe769oSEWJc8bikVFsJrN8ZQtlTo75/NGRPsrAiZMzp28jTDpiZwOlOZMiSWCqXD3Y5U4tWsVJrxAzvy256jPPxpMqp2I6sJblaEjE+qyt8/TmbtzkO8OrAjjaLKux3JOLo2rsbfe7fgq+QdvL14s9txjCkQK0LGp4nfbeSrFTt4qHcLejSv7nYck83wCxpxSasaPDt7DfGb97kdx5izZkXI/MW3a3cxZu46+ravzfALGrkdx/ggIoy5rj11qpTxdCJ7ON3tSMacFStC5k82ph3hnhnLaVWrIi/2b2djAwWwSmXCmTQoloPHT3H3jERO242sJghZETK/O3TiFLe9F094WAiTh8RSplSo25FMLlrVrsjoq9ry06Z9jJm73u04xuRbnouQiLwoIhVFJFxE5otImogM9mc4U3QyM5X7PljO1r3HmDgohugqZd2OZPKof2w0N3aux+sLNzJn1U634xiTL/k5EuqlqoeAK4DNQBPg7/4IZYre2Hnrmb92N0/0bUWXRlXdjmPy6YkrWtEuuhIPfpTE5j1H3Y5jTJ7lpwiFOV8vBz5W1YN+yGNc8FXyDiYs+JXr4+oypEt9t+OYs1A6PJSJg2IIDRWGT0vg+MkMtyMZkyf5KUL/FZG1QCwwX0SiAOtbPsit3n6IBz9OIqZeZZ65qrVdiBDEoquU5eXrO7Bu12Ee+2KF3chqgkKei5CqjgK6AnGqego4CvTzVzDjf/uOnmTY1Hgqlgnj9cGxRITZhQjBrkfz6txzcVM+S9zG+z9vdTuOMbnKz4UJ1wGnVDVDRB4HpgG1/ZbM+NXpjEzuej+R3YfTmTwkjuoVS7sdyRSSkRc15YJmUTw9czVJKQfcjmNMjvJzOu4fqnpYRLoBPYH/AJP8E8v42+jZa1iycS/PXt2WDnUrux3HFKKQEOHl6zsQVSGCEdMT2X/0pNuRjDmj/BShrE86LwemqOpXgA2tGYQ+SUjl7cWbueW8BlwbG+12HOMHVcqVYtLgGNIOp3PPh8vJyLTPh0xgyk8R2iYik4HrgdkiEpHP5f9ERCJFZJ6IbHC+VvHRpr6IJIrIchFZJSLDvV4bLSIpInIk2zL1RGSBiCwTkWQR6ePMDxeRd0VkhYisEZFHzjZ7MFuecoBHP19B18ZVeaxPS7fjGD9qF12Zp65szaL1aYyfv8HtOMb4lJ8iMgCYA/RW1QNAJAW7T2gUMF9VmwLznefZ7QDOVdUOQGdglIhkfQ41C+jkY5nHgY9UtSNwAzDRmX8dEKGqbfFc4Xe7iDQoQP6gs/vQCW6fGk/1ChFMuDGGsFDrMKO4G9ipLv1john/7QYWrNvtdhxj/iI/V8cdAzYCvUXkLqC6qs4twLb7Ae860+8CV/nY5klVzeqZ8U9HXqr6k6ru8BUVqOhMVwK2e80vJyJhQBngJHCoAPmDSvrpDG6flsCh46d546Y4IsvZmdSSQET411VtaF6jAvd9uJyUfcfcjmTMn+Tn6rh7gOlAdecxTUTuLsC2a3gVkZ1AjTNst66IJAMpwAuqut1XOy9PAYNFJBWYDWRl/ATPZeU7gK3AGFUtEX3gqypPfLGKZVsPMOa69rSsVTH3hUyxUaZUKK8PjiUjUxkxPZETp+xGVhM48nM+5lags6o+oapPAF2A23JaQES+EZGVPh5/ur9IPXfV+fzkVFVTVLUdnm6ChoqIz2LlZSDwjqpGA32AqSISgufUXQaey8obAg+IiM9xCkRkmIjEi0h8WlpaLpsLfFN/2sKH8SnceWFjLm9Xy+04xgUNqpXjpevas2LbQZ6etdrtOMb8Lj9FSPjjCjmc6Rxvr1fVnqraxsfjS2CXiNQCcL7meMLaOQJaCZyfS85bgY+cZX4ESgPVgBuBr1X1lKruBhYDcWfY1hRVjVPVuKioqFw2F9h+2rSXZ2at5uIW1XngkuZuxzEu6tW6Jnf0aMyMn7fycXyK23GMAfJXhN4GlorIUyLyFPATnnuFztZMYKgzPRT4MnsDEYkWkTLOdBWgG7Aul/VuBS52lmmJpwilOfMvcuaXw3Mkt7YA+QNe6v5jjJieSL2qZRl3QwdCQqxLnpLugUuacW6jqjz+xUpWby8xH4maAJafCxPGArcA+5zHLar6cgG2/TxwiYhswHPz6/MAIhInIm86bVriKXxJwEI8n+OscNq96HzuU1ZEUp3CCPAAcJuzzAzgZud032tAeRFZBfwCvK2qyQXIH9COn8zg9qkJnDqdyRs3xVGxdLjbkUwACAsNYfzAjlQuG84d0xM4ePyU25FMCSe5dXIoIpE5vV7cP9yPi4vT+Ph4t2Pki6oy8oPl/Dd5O28NPYcLW1R3O5IJMAlb9nH95J/o0TyKKUPi7CjZFDoRSVBVnx95eMvLkVACEO98zZqO95o2Aeb1hZuYlbSdB3s1twJkfIqtH8ljl7fkmzW7eX3RRrfjmBIsLLcGqtowLysSkdaquqrgkUxBLFi3mxfnrOXydrUY0aOx23FMALu5awMStuxnzJx1dIiuTNcm1dyOZEqgwrxlfmohrsuchU1pRxg5Yxktalbk39e2s7GBTI5EhBf6t6NRVHnunrGMnQdteDBT9AqzCNlfPBcdPnGKYVMTCAsRpgyJpWypXA9yjaFcRBivD47hxKkMRkxP4OTpTLcjmRKmMIuQddPrksxM5b4Pl/PbnqO8NiiGupFl3Y5kgkiT6hV44dp2JG49wLOz17gdx5Qw1oNlMfDyN+v5Zs1u/nF5S7o2tvP6Jv+uaFebv53XkHeWbGZmUm49YxlTeAqzCNnIWS74euUOxn/7K9fFRjO0awO345gg9kifFsTVr8KoT5PZsOuw23FMCZGfDkxjfDwaO71So6pd/BfT+LJ25yHu/yiJDnUr86+r29iFCKZAwkNDmHBjDGVLhTJ8WgJH0k+7HcmUAPk5EpqIp6ueKcAbwI/Ax8A6Eenlh2wmB/uPnuS29+IpHxHG5CGxRISFuh3JFAM1K5Xm1YEx/LbnKA9/kkxuN7MbU1D5KULbgY5Ox56xQEdgE3AJ8KI/whnfTmdkcteMRHYdTOf1IbHUqFja7UimGDm3cVUeurQFX63YwVuLN7sdxxRz+SlCzbxvRlXV1UALVd1U+LFMTp7731oW/7qXf13Vhph6fxkV3ZgCu717I3q1qsFzs9fwy+Zi3TOXcVl+itAqEZkkIhc4j4nAahGJAKwXxCLyaUIq//nhN4aeW58B59R1O44ppkSEMQPaE12lDHdOT2T3YbuR1fhHforQzcCvwL3OY5Mz7xRwYWEHM3+VlHKARz5fQZdGkTx+RSu345hirmLpcCYNjuXQiVPc/f4yTmfYjaym8OVnKIfjwKvAE8A/gFdU9ZiqZqrqEX8FNB67D5/g9qkJRJWPYOKgWMJD7RYv438ta1Xk2avbsvS3ffx7bm5DeRmTf3nu20VEegDvApvxdNFTV0SGquoi/0QzWU6ezmTEtEQOHD/Jp3d0JbJcKbcjmRLkmphoErbsZ/LCTXSsW4VL29R0O5IpRvLTwdhLQC9VXQcgIs3wDBoX649gxkNVeXLmSuK37OfVgR1pXbuS25FMCfRE31as2HaQv3+cRPOaFWhYrZzbkUwxkZ9zOuFZBQhAVdcDNlynn01bupUZP6dwR4/G9G1f2+04poSKCAtl4qAYQkOFO6YlcOyk3chqCkd+ilC8iLwpIj2cxxvYoHZ+tXTTXp6euYoezaN4sFdzt+OYEi66SlleuaEj63Yd5rHPV9qNrKZQ5KcI3QGsBkY6j9XOPOMH2w4cZ8T0ROpFen7xQ234ZRMALmgWxb0XN+PzZduYvnSr23FMMZDnz4RUNR0Y6zyMHx0/mcHtU+NJP53JlJtiqVTGznqawHH3RU1YlrKfZ2atpk2dSnSoW9ntSCaI5XokJCIrRCT5TI+iCFmSqCqjPktm1fZDvHx9B5pUr+B2JGP+JCREGDegA1EVIhgxLYF9R60DfXP28nIkdIXfU5jfvfH9Jr5cvp0HezWjZ6sabscxxqcq5Urx+uBY+k9awj0fLOOdWzrZKWNzVnI9ElLVLTk9stqJyI/+jVr8LVyfxvP/W0uftjW588ImbscxJkdtoyvxdL/WfL9hD6/M3+B2HBOkCvO2e+vKuQA27znK3e8n0qxGBf59bXsbG8gEhRvOqcu1sdGMn7+BBWt3ux3HBKHCLEJ2veZZOpJ+mtveiyckRHjjpjjKReTnHmJj3CMi/LNfG1rWqsi9Hy4nZd8xtyOZIGMdkLksM1O578PlbNpzlNdujKFuZFm3IxmTL2VKhfL64BgyVbljegInTmW4HckEkcIsQnb+6Cy8Mn8D81bv4tE+LTmvSTW34xhzVupXLcfYAR1Yue0QT89alfsCxjgKswgNKcR1lQhfr9zJK/M30D8mmr+d18DtOMYUyCWtajCiR2Nm/JzCR/EpbscxQSLPRUhEDovIoWyPFBH5XEQaqepKfwYtbtbvOswDHy2nfXQlRl/dxi5EMMXC/Zc0o2vjqvzji5Ws2n7Q7TgmCOTnSOhl4O9AHSAaeBB4H/gAeKvwoxVfB46d5Lb34ikbEcbkIXGUDg91O5IxhSIsNITxAztSpWwp7piWyMFjNuiyyVl+itCVqjpZVQ+r6iFVnQL0VtUPgSp+ylfsnM7I5O4Zy9h+4DivD46hZiW7st0UL9XKR/DaoBi2HzjOAx8vJzPTLpw1Z5afInRMRAaISIjzGABkDTxvP2V59MLXa/l+wx7+2a8NsfUj3Y5jjF/E1q/C45e35Js1u5m0cKPbcUwAy08RGoTn4oPdwC5nerCIlAHu8kO2YueLZdt44/vfGNKlPjd0qud2HGP8amjXBvRtX5uX5q5j8a973I5jAlSei5CqblLVvqpaTVWjnOlfVfW4qv6Q3w2LSKSIzBORDc7Xv5zSE5H6IpIoIstFZJWIDPd6bbRzYcQRH8vMdzpY/U5Eor1eG+psb4OIDM1v5oJYkXqQhz9NplPDSJ7o26ooN22MK0SE569pS+Oo8oycsYwdB4+7HckEoPxcHdfM+eO+0nneTkQeL8C2RwHzVbUpMN95nt0O4FxV7QB0BkaJSNbworOATj6WGQO8p6rtgGeA55y8kcCTzno6AU/6Knz+kHY4nWFT46larhQTB8UQHmr3CJuSoVxEGJMGx3LiVAYjpidy8nSm25FMgMnPX8M3gEeAUwCqmgzcUIBt9wPedabfBa7K3kBVTzrjGAFE4JVXVX9S1R0+1tsK+NaZXuBsB6A3ME9V96nqfmAecGkB8ufJydOZjJiewP5jJ5lyUxzVykf4e5PGBJQm1cvz4rXtWbb1AM/OXuN2HBNg8lOEyqrqz9nmFWSg+RpeRWQn4HPcAhGp64xblAK8oKrbc1lvEnCNM301UEFEquK5tNz7DrpUZ56vbQ4TkXgRiU9LS8vb3pzB07NW8cvm/bzQvx1t6lQq0LqMCVaXt6vFrd0a8s6SzXy5fJvbcUwAyU8R2iMijXGuhBORa/GcLjsjEflGRFb6ePTzbqeewep9XmGnqinOqbUmwFARyW2QnQeBC0RkGXABsA3IV2dWqjpFVeNUNS4qKio/i/7J9KVbmL50K7df0Ih+HXzWO2NKjFGXteCcBlUY9ekK1u867HYcEyDyU4TuBCa49WwVAAAYp0lEQVQDLURkG3AvMDynBVS1p6q28fH4EtglIrUAnK859gPvHAGtBM7PrZ2qXqOqHYHHnHkH8BSjul5No515fpGUcoCnZq7igmZRPNS7hb82Y0zQCA8NYcKNMZSLCGP41AQOn7AbWU3+itA24G1gNJ5eEuYBBbnCbKbX8kOBL7M3EJFo5xJwnIsIugHrclqpiFQTkaz9eoQ/enOYA/QSkSrOuno58/yiec0K/K1bQ8bf0NFGnDTGUaNiaSbc2JEt+47x8KfJeE6CmJIsP0XoS6AvngsTtgNHgKMF2PbzwCUisgHo6TxHROJE5E2nTUtgqYgkAQuBMaq6wmn3ooikAmVFJFVEnnKW6QGsE5H1eD5nGg2gqvuAfwK/OI9nnHl+UTo8lEcua0mlsuH+2oQxQalLo6o81Ls5s1fs5D8//OZ2HOMyyet/IiKyUlXb+DlPwImLi9P4+Hi3YxhTrKgqt09NYP7a3cy4rQudGlrvIcWNiCSoalxu7fJzJLRERNoWIJMxxgCeG1nHDGhPvciy3Pl+IrsPn8h9IVMs5acIdQMSRGSd0xvBCufSaWOMybeKpcOZNDiGwydOcdf7y2xE1hIqP0XoMqApng/0+wJXOF+NMeastKhZkeeuacvPv+2jz/jvSdjit49pTYDKT99xW3w9/BnOGFP8Xd0xmvf+1on0U5lc+/qPPDNrNcdP2lFRSWGdmBljXNe9WRRz7uvOoM71eGvxb1z6yiJ+3LjX7VimCFgRMsYEhPIRYfzrqrbMuK0LqjDwjZ94/IsVHEkvSO9gJtBZETLGBJRzG1fl63vP59ZuDZm+dCu9xy1i4fqC9eFoApcVIWNMwClbKox/XNGKT4Z3pXR4CEPf+pmHPkni4HHr6qe4sSJkjAlYsfWr8NXI87mjR2M+TdzGJWMXMm/1LrdjmUJkRcgYE9BKh4fy8KUt+HxEVyLLleK29+K554Nl7Dt60u1ophBYETLGBIV20ZWZeVc37u3ZlK+Sd3DJ2IV8lZzjaDImCFgRMsYEjVJhIdzbsxmz7u5G7cpluPP9RO6YlmDd/gQxK0LGmKDTslZFPh/RlYcubc78tbvpNW4Rny9LtaEhgpAVIWNMUAoLDWFEjybMHtmNRtXKcd+HSdz6bjw7Dh53O5rJBytCxpig1qR6BT4e3pV/XNGKJRv30GvsIj74easdFQUJK0LGmKAXGiLc2q0hc+7tTqvaFRn12QqG/OdnUvYdczuayYUVIWNMsVG/ajlm3NaFf17VhmVb99P75UW89+NmMjPtqChQWREyxhQrISHCkC71mXNfd2LrV+GJL1dxw5Sf+G3PUbejGR+sCBljiqXoKmV572+dePHadqzZeYhLX17EG4s2kWFHRQHFipAxptgSEQbE1eWb+y/g/KbVGD17Df0nLWHDrsNuRzMOK0LGmGKvRsXSvHFTHK/c0IEte49y+fgfeG3Br5zKyHQ7WolnRcgYUyKICP061GHufRdwSasa/HvOOq56bTGrtx9yO1qJZkXIGFOiRFWI4LVBMUwaFMOuQye4csIPjJ23npOn7ajIDVaEjDEl0mVtazHvvgu4sn1txs/fQN9XfyAp5YDbsUocK0LGmBKrSrlSjL2+A2/dHMfB46e4euJinvvfGk6cynA7WolhRcgYU+Jd1KIGc+/vzoC4ukxeuIk+478nfvM+t2OVCFaEjDEGqFg6nOf7t2PqrZ1IP5XJdZN/5OlZqzh28rTb0Yo1K0LGGOPl/KZRzL2vOzd1qc/bizfT++VFLNm4x+1YxZYVIWOMyaZcRBhP92vDh8O6ECrCjW8s5bHPV3D4xCm3oxU7VoSMMeYMOjeqyv/u6c5t5zdkxs9b6T1uEd+t2+12rGLFipAxxuSgTKlQHru8FZ/c0ZWyEWHc/PYvPPhxEgeP2VFRYbAiZIwxeRBTrwpfjezGnRc25vNl2+g5biFzV+10O1bQc60IiUikiMwTkQ3O1yo+2tQXkUQRWS4iq0RkuNdro0UkRUSO+Fhmvogki8h3IhLtzO8gIj8660kWkev9v5fGmOIkIiyUv/duwZd3nkfVcqUYNjWBu2csY++RdLejBS03j4RGAfNVtSkw33me3Q7gXFXtAHQGRolIbee1WUAnH8uMAd5T1XbAM8BzzvxjwE2q2hq4FHhZRCoX2t4YY0qMNnUqMfOubtx/STO+XrmDXuMW8d/k7Tak+Flwswj1A951pt8FrsreQFVPqmrWvxgReOVV1Z9UdYeP9bYCvnWmFzjbQVXXq+oGZ3o7sBuIKoT9MMaUQKXCQhh5cVP+e/f5RFcpw13vL2P4tAR2Hz7hdrSg4mYRquFVRHYCNXw1EpG6IpIMpAAvOAUkJ0nANc701UAFEamabZ2dgFLAxjNsc5iIxItIfFpaWt72xhhTIjWvWYFP7+jKqMtasGBdGpeMXcSnCal2VJRH4s9vlIh8A9T08dJjwLuqWtmr7X5V/cvnQl6v1wa+APqq6i6v+UdUtXy2dhOAhsAioD/QRlUPOK/XAr4DhqrqT7ntQ1xcnMbHx+fWzBhj2Jh2hIc+SSZhy34ubB7Fs9e0pValMm7HcoWIJKhqXK7t3KrWIrIO6KGqO7IKg6o2z2WZt4DZqvqJ17w/FaFs7csDa1U16+KEingK0LPe68iJFSFjTH5kZCrv/biZF79eR1iI8OjlLbnhnLqIiNvRilRei5Cbp+NmAkOd6aHAl9kbiEi0iJRxpqsA3YB1Oa1URKqJSNZ+PQK85cwvBXyO56KFPBUgY4zJr9AQ4ZbzGjLn3u60qVOJRz5bweD/LCVl3zG3owUkN4vQ88AlIrIB6Ok8R0TiRORNp01LYKmIJAELgTGqusJp96KIpAJlRSRVRJ5ylukBrBOR9Xg+ZxrtzB8AdAdudi75Xi4iHfy+l8aYEqle1bJM/7/OjL66DUkpB+k1bhHvLP6NzEz7rMiba6fjgoWdjjPGFNS2A8d59LMVLFyfxjkNqvBC/3Y0ivL5KUKxEQyn44wxpkSoU7kM79xyDmOua8+6nYe57JXvmbJoIxl2VGRFyBhjioKIcG1sNN/cfwHdm0Xx7Oy1XDNpCet3HXY7mqusCBljTBGqXrE0U4bE8urAjqTsO8YV439gwrcbOJWR6XY0V1gRMsaYIiYi9G1fm3n3dadX6xqMmbuefhMWs2r7QbejFTkrQsYY45Kq5SOYcGMMrw+OZffhdPpNWMxLc9eRfjrD7WhFxoqQMca47NI2Nfnm/u7061CHV7/9lb6v/sDylANuxyoSVoSMMSYAVC5bipcGtOftW87h8InTXDNxMc/NXsOJU8X7qMiKkDHGBJALm1dnzn3duf6cekxetInLXvmeXzbvczuW31gRMsaYAFOxdDjPXdOW6f/XmVMZmQyY/CNPzVzF0fTTbkcrdFaEjDEmQJ3XpBpz7u3O0HMb8O6Pm+n98iIW/7rH7ViFyoqQMcYEsHIRYTx1ZWs+uv1cSoWGMOjNpTzy2QoOnTjldrRCYUXIGGOCwDkNIpl9z/nc3r0RH/6yld7jFrFg3W63YxWYFSFjjAkSpcNDeaRPSz4bcR4VSodxy9u/cP9Hyzlw7KTb0c6aFSFjjAkyHepWZtbd3Rh5URNmLt/OJeMWMWfVTrdjnRUrQsYYE4QiwkK5v1dzvrzrPKLKR3D71ATuej+RvUfS3Y6WL1aEjDEmiLWuXYkv7zqPB3s1Y+6qXVwybhEzk7YTLGPFWREyxpggFx4awl0XNeW/I7tRN7IsI2csY9jUBHYfOuF2tFxZETLGmGKiWY0KfHZHVx7t04JF69PoOXYhnySkBvRRkRUhY4wpRkJDhGHdG/O/e86nec0KPPhxEje//QvbDhx3O5pPVoSMMaYYahRVng+HncvTV7bml8376D1uEdOXbgm4oyIrQsYYU0yFhAhDuzZgzr3daV+3Eo99vpJBby5l695jbkf7nRUhY4wp5upGlmXarZ15/pq2rEg9SO+XF/H24t/IzHT/qMiKkDHGlAAiwg2d6jHnvu50aRTJ07NWM2Dyj2xMO+JqLitCxhhTgtSuXIa3bj6HsQPas2H3Efq88j2TF27kdEamK3msCBljTAkjIlwTE828+7vTo3kUz/1vLf0nLWHdzsNFnsWKkDHGlFDVK5Tm9cGxTLixI6n7j3PFq98zfv4GThXhUZEVIWOMKcFEhCva1Wbufd25rE0txs5bz5UTFrNy28Ei2b4VIWOMMVQtH8H4gR2ZMiSWvUfS6ffaYsbNW+/37Yb5fQvGGGOCRq/WNencsCr/+mp1kdzYakXIGGPMn1QqG86/r2tfJEXITscZY4zxSUT8vg0rQsYYY1zjWhESkUgRmSciG5yvVXy0qS8iiSKyXERWichwr9dGi0iKiBzxscx8EUkWke9EJDrb6xVFJFVEJvhv74wxxuSFm0dCo4D5qtoUmO88z24HcK6qdgA6A6NEpLbz2iygk49lxgDvqWo74BnguWyv/xNYVAj5jTHGFJCbRagf8K4z/S5wVfYGqnpSVbMGTI/AK6+q/qSqO3ystxXwrTO9wNkOACISC9QA5hY4vTHGmAJzswjV8CoiO/EUh78QkboikgykAC+o6vZc1psEXONMXw1UEJGqIhICvAQ8WPDoxhhjCoNfi5CIfCMiK308+nm3U891gD6vBVTVFOfUWhNgqIj4LFZeHgQuEJFlwAXANiADGAHMVtXUPOQeJiLxIhKflpaW+44aY4w5K369T0hVe57pNRHZJSK1VHWHiNQCdueyru0ishI4H/gkp3Y4R0IiUh7or6oHRORc4HwRGQGUB0qJyBFV/ctnUao6BZgCEBcX5/6AG8YYU0yJW0O9isi/gb2q+ryIjAIiVfWhbG2inTbHnavnluIpKiu82hxR1fJez6sB+1Q1U0RGAxmq+kS29d4MxKnqXXnImQZsOfs9pRqwpwDLB4rish9g+xKIist+gO1LlvqqGpVbIzd7THge+EhEbsXzR34AgIjEAcNV9f+AlsBLIqKAAGOyCpCIvAjcCJQVkVTgTVV9CugBPOcsswi4syAh8/JNzImIxKtqXEHWEQiKy36A7UsgKi77AbYv+d6GW0dCJUVx+YEsLvsBti+BqLjsB9i+5Jf1mGCMMcY1VoT8b4rbAQpJcdkPsH0JRMVlP8D2JV/sdJwxxhjX2JGQMcYY11gRKiQi8paI7HbuZcqal2snrYHoDPvylIhsczqTXS4ifdzMmBdObxsLRGS10wHuPc78oHtfctiXYHxfSovIzyKS5OzL0878hiKyVER+FZEPRaSU21lzk8O+vCMiv3m9Lx3czpoXIhIqIstE5L/Oc7+/J1aECs87wKXZ5uWlk9ZA9A5/3ReAcarawXnMLuJMZ+M08ICqtgK6AHeKSCuC8305075A8L0v6cBFqtoe6ABcKiJdgBfw7EsTYD9wq4sZ8+pM+wLwd6/3Zbl7EfPlHmCN13O/vydWhAqJqi4C9mWbnWsnrYHoDPsSdFR1h6omOtOH8fxy1SEI35cc9iXoqEfWECzhzkOBi/ijN5RgeV/OtC9Bx+kc4HLgTee5UATviRUh/8pTJ61B5C7xjNP0VjCcwvImIg2Ajnh63Qjq9yXbvkAQvi/OaZ/leLrrmgdsBA6o6mmnSSpBUmSz74uqZr0vo533ZZyIRLgYMa9eBh4CMp3nVSmC98SKUBHJqZPWIDEJaIznlMMOPD2SBwWnD8FPgXtV9ZD3a8H2vvjYl6B8X1Q1wxknLBrPuGAtXI501rLvi4i0AR7Bs0/nAJHAwy5GzJWIXAHsVtWEot62FSH/2uV0zkpeOmkNZKq6y/llywTewPeAggFHRMLx/NGerqqfObOD8n3xtS/B+r5kUdUDeMb9OheoLCJZXYlF4+kBP2h47culzulTdcZDe5vAf1/OA64Ukc3AB3hOw71CEbwnVoT8ayYw1JkeCnzpYpYCyfqj7bgaWHmmtoHCOaf9H2CNqo71eino3pcz7UuQvi9RIlLZmS4DXILnM64FwLVOs2B5X3zty1qvf3IEz+coAf2+qOojqhqtqg2AG4BvVXUQRfCe2M2qhUREZuDpPLUasAt4EvgC+Aioh9NJq6oG/Af+Z9iXHnhO+SiwGbj9DCPbBgwR6QZ8D6zgj/Pcj+L5LCWo3pcc9mUgwfe+tMPzIXconn+EP1LVZ0SkEZ7/wiOBZcBgr5GVA1IO+/ItEIWn4+XleDplPnLmNQUOEekBPKiqVxTFe2JFyBhjjGvsdJwxxhjXWBEyxhjjGitCxhhjXGNFyBhjjGusCBljjHGNFSFjjDGusSJkipSIXCkiBe612ukq/9pc2twsIrXPcv3PiEjPs1juZhGZcDbbdJaf4fQ3dp/z/GUR6Z6P5WuLyCe5t8zz+p4SkQfP8NoS52sD8Rr2I1ub70QkrrDyFERefmbOsNwVIvKMPzIZK0KmiKnqTFV9vog2dzNwVkVIVZ9Q1W8KN07ORKQmcI6qtlPVcSJSFeji9GqeJ6q6XVXz/Yf2bKhq16LYTgD4CugrImXdDlIcWREyhcb5j3it8x/nehGZLiI9RWSxeAaQ6+R9pOC0Gy8iS0RkU07/pYrHBBFZJyLfANW9XntCRH4RkZUiMsVpey0QB0wXz6BiZXy1y2F7v//XLCKbReRpEUkUkRUikqfONp0uXT51tvmLiJznzO8kIj+KZ/CwJSLS3FlkLlDHyXs+0B/42mt9m0XkOef1eBGJEZE5IrJRRIZ7vQcrnembReQzEfna+f6/mEveS519TBKR+V4vtXKOaDaJyEiv9n/pAcD5Pn8gImtE5HOgTA7bC3W+zyud72vW0d9tzvcryfn+lXXmvyMik0TkJydLD/H0HL5GRN7xziWenqtXich8EYnyse1YEVkoIgnO9zCrm52R4hk4MFlEPoDfO7n9Drgip++fOUuqag97FMoDaIBn8LW2eP7BSQDewtN1ST883RjdDExw2r8DfOy0bQX8msO6r8HT5X8onqObA8C1zmuRXu2mAn2d6e+AOK/XfLY7w/be8Vr/ZuBuZ3oE8GYOy3nv3/tAN2e6Hp5+3wAqAmHOdE/gU6/v30qvdb3rndHJcYczPQ5IBirg6R5mV/Z1OFk2AZWA0ni6KKp7htxRQArQ0Pt7BTwFLAEi8HTjtBcId1474mOb9wNvOdPtnJ+HuDNsMxbP0AdZzys7X6t6zfuX1/f+HTxdyGT9PB3izz9rHZx2Cgxypp/gzz9v1+IZ82cJEOXMv94r83YgwjuPMz0IeNXt37Hi+MjqHdWYwvKbqq4AEJFVeEYwVRFZgeePVXZfqKcH6NUiktO4Pt2BGaqaAWwXT99cWS4UkYeAsnj6uFoFzPKxjry28yWrB+4EPAUxL3riOYrIel5RPEMxVALeFZGmeP5ghp9h+VpAWrZ5M52vK4Dy6hng7rCIpIvTkWY281X1IICIrAbq4yk22XUBFqnqbwD65770vlJPf2HpIrIbz/hLqWfI3B0Y76wjWUSSz9AOPAWykYi8iueU11xnfhsR+RdQGSgPzPFaZpbXz9OubD9rDfD005YJfOi0n8Yf712W5kAbYJ7z3oTiGQYDPIV9uoh8geefpiy7OctTuyZnVoRMYfPu3DDT63kmvn/evNuf8fTYmYhIaWAinv+2U0TkKTz/9Z9Vuxxk5cwg7783IXg+0zmRLcsEYIGqXi2eAeq+O8Pyx31k9P5+Zv9e5/b9zU/2wl7HX6jqfhFpD/QGhgMDgL/hOWK5SlWTRORmPJ3nZs+S1/2Hv44XJcAqVT3XR9vL8RTSvsBjItJWPYO6lcbzfphCZp8JmWCxCLje+RyhFnChMz/rj/Qe5yjD+3Olw3hOV+XWzl/mAndnPRGRDs5kJf4Yl+XmHJZfAzTxS7K/+gnoLiINAUQk8izXswi40VlHGzyn5HwSkWpAiKp+CjwOxDgvVQB2iGf8pEFnkSGEP97fG4Efsr2+DogSkXOdHOEi0lpEQvCcrlyAZxC6SniOxACaEeDDMQQrK0ImWHwObABWA+8BP8LvA4m9gecPxBzgF69l3gFeF8/Qy+k5tPOXkUCc8yH3ajz/7QO8CDwnIsvI+ajiK/58FOA3qpoGDAM+E5Ek/jidlV+TgPIisgZ4Bs/pyzOpA3znvD/T8IxGCvAPPMNtLAbWnkWGo3hGOF2JZ3C2P11eraon8RSpF5x9XQ50xXNabppzqm8ZMN75+QLPPz1fnUUWkwsbysGYACYiPwBXeP0xNLkQkSOqWj73lnleXw3gfVW9uLDWaf5gRciYACYinYHjqprTB/zGix+K0DnAKVVdXljrNH+wImQCioi0xXP5tLd0Ve3sp+29BpyXbfYrqvp2LsvdAtyTbfZiVb2zMPP5g4gsxXPJtbchWVeaFZdtmuBgRcgYY4xr7MIEY4wxrrEiZIwxxjVWhIwxxrjGipAxxhjXWBEyxhjjmv8Hwy0rYSLmkwQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "plt.plot(min_child_samples_s, test_means)\n",
    "plt.xlabel( 'min_data_in_leaf(min_child_samples)' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最优值为min_child_samples=20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "4)colsample_bytree\n",
    "用来控制每颗树随机采样的列数的占比。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   39.8s\n",
      "[Parallel(n_jobs=5)]: Done  16 out of  20 | elapsed:  1.3min remaining:   19.7s\n",
      "[Parallel(n_jobs=5)]: Done  20 out of  20 | elapsed:  1.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "        ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'colsample_bytree': [0.5, 0.6, 0.7, 0.8]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'num_leaves':70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "colsample_bytree_s = [i/10.0 for i in range(5,9)]\n",
    "tuned_parameters = dict( colsample_bytree = colsample_bytree_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.321565357829197\n",
      "{'colsample_bytree': 0.8}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAELCAYAAAAP/iu7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VdW5//HPl5kAAhEENAxWUaCCohFEbWtb2jrUoVpnFBDEoZPtbS2t/np7e/Ve9No6VG2rICLO0oq22EEpap2AgAwOIBZUEARkEglj8vz+ODt6iIEQkpNzknzfr9d5ZQ9r7/MstubJXnvttRQRmJmZZVKjbAdgZmb1n5ONmZllnJONmZllnJONmZllnJONmZllnJONmZllnJONmZllnJONmZllnJONmZllXJNsB5ArOnToED169Mh2GGZmdcqsWbM+jIiOlZVzskn06NGDoqKibIdhZlanSHp3T8q5Gc3MzDLOycbMzDLOycbMzDLOycbMzDLOycbMzDLOycbMzDLOycbMzDLOycbMrIHaXlLKHdPepnjbjox/l1/qNDNrgDZt3cGVD8zmubdWU9C+JacfcUBGv8/JxsysgVm1cQuX3DuTN1dsZMyZfTOeaMDJxsysQXl71UaG3jOTtZu2MfbiQr7ca79a+V4nGzOzBmLmO2sZOaGIpo3FI5cdQ7+CdrX23U42ZmYNwFPzV3DVI3MoaNeSe4cPoNu+ebX6/U42Zmb13LgXlnDdlDc4slt7xl5cSPtWzWo9BicbM7N6qrQ0uP6pNxn3whJO/HxnbjnvCFo0bZyVWLL2no2kfElPS1qU/GxfQZnukmZLmiPpdUmXJ9vzJE2RtCDZPibtmB9JekPSPElTJXWvzXqZmeWCLdtL+N5DrzLuhSUMO7YHd1x4ZNYSDWT3pc7RwNSI6AlMTdbLWwEMiogjgIHAaEn7J/tuioheQH/gOEknJdtfBQojoh8wCbgxk5UwM8s164u3cdG46UyZv4JrT+nNf57ah8aNlNWYsplsTgcmJMsTgDPKF4iIbRGxNVltThJvRBRHxLSyMsBsoCBZnxYRxckxr5RtNzNrCJauLeas373E3KUb+O35/Rn5hc8hZTfRQHaTTaeIWJEsfwB0qqiQpK6S5gFLgRsiYnm5/e2AU0ndHZU3AvhrzYVsZpa7Xnt/A2f+7iVWb9zKxBEDOPXw/Ss/qJZktIOApGeAzhXsuiZ9JSJCUlR0johYCvRLms8mS5oUESuT8zcBHgJui4jF5b57CFAIfGk38Y0CRgF069Ztj+tlZpZrnl24iisfmE37vGY8OHIgPTu1yXZIO8losomIwbvaJ2mlpC4RsUJSF2BVJedaLuk14AuknsUA3AUsiohbyp17MKmE9qW0ZriKznlXcg4KCwsrTHZmZrnu0ZlL+dnj8zm0UxvGDz+aTvu0yHZIn5HNZrQngaHJ8lDgifIFJBVIapkstweOBxYm69cBbYGryh3TH/gDcFpE7DaBmZnVZRHBzU+/xdV/nMexB+3Lo5cPyslEA9lNNmOAr0laBAxO1pFUKGlsUqY3MF3SXOA5Uj3Q5ksqIHXn0gco6xo9Mjnm/4DWwGPJ9idrsU5mZrVie0kpV0+ax61TF/Htowq4Z9jRtG6eu69OKsKtR5BqRisqKsp2GGZmlfo4mR7g+bdW8/2v9uSHg3tmrceZpFkRUVhZudxNg2Zm9hmrPtrC8HtnsuCDjdxwVl/OPbpudG5ysjEzqyPKpgdYV7yNsUML+fKhtTM9QE1wsjEzqwNmLFnLpfcV0bRxIx4ZNYi+BW2zHVKVONmYmeW4KfNW8MNH51DQviUThg+ga37tTg9QE5xszMxy2Nh/Leb6p97M6vQANcHJxswsB5WWBtdNeZN7XlzCSYd15uZzszc9QE1wsjEzyzFbtpfwo0fn8NT8Dxh+XA+uPSX7ozZXl5ONmVkOWV+8jUvvK2LmO+u49pTejPzC57IdUo1wsjEzyxFL1xYzbPwMlq7dzO0X9Oeb/XJn1ObqcrIxM8sB85dtYPi9M9m2o4SJIwYw8HP7ZjukGuVkY2aWZdMWruI7yfQAD12ae9MD1AQnGzOzLHpk5nv8/PHX6NW5DeOHHc1+OTpqc3U52ZiZZUFEcMszi7h16iK+eEhH7rzwyJwetbm66m/NzMxy1PaSUn7+p/k8NmsZZx9VwP+c2ZemjbM540vmOdmYmdWi9OkBfvDVnlyVxekBapOTjZlZLamr0wPUhKzdt0nKl/S0pEXJz/YVlOkuqWwmztclXZ5sz5M0RdKCZPuYtGMulzQ/OeYFSX1qs15mZhV5e9VGvnXnSyz5cBNjhxY2qEQD2Z0WejQwNSJ6AlOT9fJWAIMi4ghgIDBaUtlbTjdFRC+gP3CcpJOS7Q9GRN/kmBuB32S0FmZmlZixZC1n/e5ltu4o5ZFRg+rUPDQ1JZvJ5nRgQrI8ATijfIGI2BYRW5PV5iTxRkRxREwrKwPMBgqS9Y/STtEK8LzXZpY1U+atYMi46ezbuhmPX3lsnZuHpqZk85lNp4hYkSx/AHSqqJCkrsAU4GDgJxGxvNz+dsCpwK1p274D/AhoBnyl5kM3M6tc2fQAR3Vrz911eHqAmpDRZCPpGaBzBbuuSV+JiJBU4R1IRCwF+iXNZ5MlTYqIlcn5mwAPAbdFxOK0Y+4A7pB0AXAtMHQX8Y0CRgF069aw2k/NLHNKSoPrprzB+BffqRfTA9QERWSnlUnSQuCEiFghqQvwbEQcWskx9wBPRcSktPWPI+L7uyjfCFgXEZXetxYWFkZRUVGV62Fmlm7L9hJ++Mgc/vraB1xy3IFcc0rvOj89wO5ImhURhZWVy+Yzmyf59I5jKPBE+QKSCiS1TJbbA8cDC5P164C2wFXljumZtnoKsKjGIzczq8C6TdsYMnY6f3v9A649pTe/OLXuz0NTU7L5zGYM8KikEcC7wDkAkgqByyNiJNAb+HXSxCZSPdDmSyog1RS3AJidvBB1e0SMBb4raTCwHVjHLprQzMxq0tK1xQwdP4Nl6zZz+/lHckq/LtkOKadkrRkt17gZzcz2Vtn0ANtLSrn74kIGHJif7ZBqzZ42o3kEATOzakifHuDhUQM5eL/6Nz1ATXCyMTPbSw1leoCa4GRjZlZFEcHNzyzitgYyPUBN8L+OmVkVNMTpAWqCk42Z2R5qqNMD1AQnGzOzPdCQpweoCU42ZmaVeHvVRobeM5N1xdsYO7SwQY7aXF1ONmZmuzFjyVpGTphJsyaNeWTUoAY7anN1OdmYme3ClHkr+OEjcyjIb8mE4QPomp+X7ZDqLCcbM7MKjP3XYq6b8iaF3dszdmgh7fIa7vQANcHJxswsjacHyAwnGzOzRPnpAa49pTeNPGpzjXCyMTMjNT3ApfcVMeu9dVx7Sm9GfuFz2Q6pXnGyMbMGz9MDZJ6TjZk1aOnTA9w/YmCDmh6gNjnZmFmD5ekBak/WRo+TlC/paUmLkp/tKyjTXdJsSXMkvS7p8mR7nqQpkhYk28dUcOxZkiKZ+dPMbCcPz3iPkROKOLBDKx6/8lgnmgzL5lClo4GpEdETmJqsl7cCGBQRRwADgdGS9k/23RQRvYD+wHGSTio7SFIb4AfA9ExWwMzqnojgN0+/xeg/zee4gzvwyGWDPA9NLchmsjkdmJAsTwDOKF8gIrZFxNZktTlJvBFRHBHTysoAs4GCtEP/G7gB2JKZ0M2sLtpeUspPJs3jtqmLOPuoAsYNLfQ8NLUkm8mmU0SsSJY/ADpVVEhSV0nzgKXADRGxvNz+dsCppO6OkHQk0DUipmQscjOrcz7euoNL7p3JpFnLuGpwT278dj/PQ1OLMprSJT0DdK5g1zXpKxERkqKic0TEUqBf0nw2WdKkiFiZnL8J8BBwW0QsltQI+A0wbA/jGwWMAujWzcOFm9VXqz7awrDxM1m4ciM3ntWPc47umu2QGpyMJpuIGLyrfZJWSuoSESskdQFWVXKu5ZJeA74ATEo23wUsiohbkvU2wGHAs8mERp2BJyWdFhFFFZzzruQcFBYWVpjszKxuS58eYNzQQk7w9ABZkc17yCeBocnyUOCJ8gUkFUhqmSy3B44HFibr1wFtgavKykfEhojoEBE9IqIH8ApQYaIxs/pv+uI1nHnnS2zdUcqjlw1yosmibCabMcDXJC0CBifrSCqUNDYp0xuYLmku8BypHmjzJRWQaorrA5R1jR5Z+1Uws1z1l3nLuWjcDDq0ac7jVx7LYQd4HppsUoRbjyDVjFZU5Bsgs7ouIhj3whJPD1BLJM2KiErfZ3SfPzOrN9KnBzi5b2d+c46nB8gVTjZmVi+kTw8w4vgDueZkTw+QS5xszKzOW7dpGyPvK2K2pwfIWU42ZlanLV1bzNB7ZrBsvacHyGV73BtN0o2S9pHUVNJUSaslDclkcGZmuzNv2Xq+deeLrNm0jftHDHSiyWFV6fr89Yj4CPgm8A5wMPCTTARlZlaZaQtWcd5dr9C8SWP+eMUgz0OT46rSjFZW9hTgsYjYkLylb2ZWqx6e8R7XTH6N3l3acM+wo9mvjUdtznVVSTZ/kbQA2AxcIakjHlXZzGpRRHDzM4u4beoivnRIR+648EiP2lxH7PFViojRkm4ENkREiaRNpKYJMDPLuO0lpfzsT/OZNGsZ5xQWcP23+nrU5jqkKh0Ezga2J4nmWuB+YP9KDjMzq7aNW7bvND3ADWd5eoC6pipX6/9FxEZJx5May2wc8LvMhGVmlrLyoy2c+4dXeOnfa7jxrH5cNfgQ/Ly47qlKsilJfp4C3JVMTuYBh8wsYxat3MiZd77Eu2s2MW5ooeehqcOq8mTtfUl/AL4G3CDpk2mazcxq2vTFa7j0viKaN23MI5cN8qjNdVxVksU5wN+Bb0TEeiAfv2djZhlQNj1AxzbN+dMVnh6gPqhKb7RiSf8GviHpG8C/IuIfmQvNzBqaiGDsv5Zw/VNvcnSP9tx9sacHqC+q0hvtB8ADwH7J535J38tUYGbWsJSUBv/15ze4/qk3OblvZyaOGOhEU49UpRltBDAwIn4REb8AjgEu3dsvlpQv6WlJi5Kf7Sso011S2Uycr0u6PNmeJ2mKpAXJ9jFpxwxLxm2b4xk8zeqGLdtL+M4Ds7n3pXcYcfyB3H7+kZ6Hpp6pSrIRn/ZII1muTv/D0cDUiOgJTE3Wy1sBDIqII4CBwGhJZe/23BQRvYD+wHGSTko77pGIOCL5jMXMcta6Tdu4cOx0/v7GB/y/b/bh/32zj+ehqYeq0httPDBd0uPJ+hmk3rXZW6cDJyTLE4BngZ+mF4iIbWmrn/R+i4hiYFpZGUmzgYJqxGJmWZA+PcAdFxzJyX09anN9tcd3NhHxG2A4sDb5DI+IW6rx3Z0iYkWy/AHQqaJCkrpKmgcsBW6IiOXl9rcDTiV1d1TmLEnzJE2S5I75ZjkofXqAB0YOdKKp5yq9s5GUPm73O8nnk30RsXY3xz4DdK5g1zXpKxERkqKic0TEUqBf0nw2WdKkiFiZnL8J8BBwW0QsTg75M/BQRGyVdBmpu6av7CK+UcAogG7duu2qGmZWw6YtWMWVD8wmv1UzHh41gIP3a53tkCzDFFHh7/hPC0hLgODT5zNlB4hUntir+VclLQROiIgVkroAz0bEoZUccw/wVERMSlv/OCK+v4vyjYG1EVFpJ/3CwsIoKiqqcj3MrGoemvEe13p6gHpD0qyIKKysXKV3NhFx4B5+4ecj4vU9KZt4EhgKjEl+PlHBOQuANRGxOemtdjxwc7LvOqAtMLLcMV3SmudOA96sQkxmliERwc1Pv8Vt/3ybLx3SkTsvPJJWnh6gwajJKz0ROLIK5ccAj0oaAbxLaoQCJBUCl0fESKA38OukiU2keqDNT5LQNcACYHYyKN/tSc+z70s6DdhB6tnSsJqonJntmS3bS1i6tph31xTz3tpPP0s+3MSSDzdxbmFXrvvWYR61uYGptBltj08kvRoR/WvkZFngZjSzPRMRrP5462cTSrK8auPWncq3ataYbvu2olt+S44/uANDjunuUZvrkRprRquCmslaZpZ1W7aXsGzd5iShbOK9tZt5b20xS5PEsnn7p6/cSdB5nxZ0y8/jS4d0pFt+Ht32zUv9zM8jv1UzJxer0WRjZnVERLB20zbeLUsga4p5N0kkS9cW88FHW0hv9GjZtPEnSeT4nh12SigHtGvpt/2tUjWZbLZVXsTMasu2HaW8v34z767Z9MkdyXtJ09fStcVs2layU/lO+zSnW34exx5UlkxaJncnrejQ2ncnVj17nGwkVfTwfwPwbkTsiIhjai4sM6tMRLC+ePtOD+HfS3uGsmLDZkrT7k6aN2n0SdPWMZ/bl+5pTV1d8/N8d2IZVZU7mztJ9TabR6pn2GHA60BbSVd4ugGzmre9pJTl6zdXmEzeW1vMxi07dirfoXVzuu+bx4AD8+man0f3tOaujq2be8wxy5qqJJvlwIiyd2kk9QF+BVwN/AlwsjHbCxs2b6+gq/Am3ltbzPL1WyhJuz1p1rgRBfmp5q3C7u3pmtyZdN+3FV3zW5LXzI9hLTdV5b/MQ9Jf2oyINyT1iojFbss127UdJaWs2LAllVAqaPLasHn7TuX3bdWMrvl59O/anjOOyEtLKHl0atPCdydWJ1Ul2bwu6XfAw8n6ucAbkpoD23d9mFn9t3HL9p26BpfdpSxdW8yydZvZkXZ30rSxKGifSiKHd21L9/xWnySUrvktadOiaRZrYpYZVUk2w4ArgauS9ReBH5NKNF+u2bDMcktJabDyoy2f9OR6b+3OXYXXbtq5M2a7vKZ0z8/jsAPacnLfLjt1Fe7StiWNfXdiDcweJ5tkfLLfkno2E8DCiCi7o/k4E8GZ1aZNW3ewdN1nH8K/tyZ1d7KtpPSTso0biQPapZ6dnHhY5096dZX17Grb0ncnZumq0vX5BFLD9b9DqjdaV0lDI+L5zIRmVrNKS4NVG7emJZFNOyWVDz/e+e6kTYsmdN83j15d2vD1z++cUPZv14ImHtvLbI9VpRnt18DXI2IhgKRDSM0lc1QmAjOrjumL1/D68o92SiZL1xazdcendyeNBPsndyeDe3dKdRVOe/ekXV6zLNbArH6pSrJpWpZoACLiLUluK7CcEhH8+h9vcfu0t4FPB4E8uGNrvtJrv0/fPcnPY/92LWnWxHcnZrWhKsmmSNJY4P5k/ULAwyRbzigtDX7559e57+V3Oe/orlx9Yi/a5zX1MCtmOaAqyeYK4DtA2ayY/yI1qoBZ1m0vKeUnj81l8pzlXPbFzzH6pF5OMmY5pCq90bYCv0k+Zjljy/YSvvPAbKYuWMXVJx7KlSccnO2QzKycSpONpPnsZq6aiOi3N18sKR94BOhBqofbORGxrlyZ7sDjQCOgKfDbiPi9pDzgMeAgoAT4c0SMTjvuHOCXSdxzI+KCvYnRct/GLdsZOaGIGe+s5bozDmPIMd2zHZKZVWBP7my+maHvHg1MjYgxkkYn6z8tV2YFMCgitkpqDbwm6UlgPakpoqdJagZMlXRSRPxVUk/gZ8BxEbFO0n4Zit+ybM3HWxk6fgYLVmzk1vP6c9rh+2c7JDPbhUqTTUS8uycnkvRyRAyqwnefDpyQLE8AnqVcsomI9BcfmpO6wyEiioFpZWUkzQYKknKXAneU3SVFxKoqxGR1xPL1m7lo3HSWrdvM3RcX8uVe/pvCLJfVZL/PFlUs3ykiViTLHwCdKiokqaukecBS4IaIWF5ufzvgVGBqsukQ4BBJL0p6RdKJuwpA0ihJRZKKVq9eXcXwLVsWr/6Ys3//Mqs+2srEEQOdaMzqgJocj/wzz3UkPQN0rqDsNTsdGBGSKnwuFBFLgX6S9gcmS5oUESuT8zch9WLpbRGxODmkCdCT1F1TAfC8pL4Rsb6Cc98F3AVQWFi4y+dSljteX76BoffMIAIeGnUMhx3QNtshmdkeyOjkFxExeFf7JK2U1CUiVkjqAuy2uSsilkt6DfgCMCnZfBewKCJuSSu6DJiejNu2RNJbpJLPzOrUxbJv5jtrueTembRp3oSJIwdyUMfW2Q7JzPZQTTajVfWlhieBocnyUOCJz5xQKpDUMlluDxwPlA2Xcx3Qlk9HoS4zmeRZkKQOpJrVFmN12rSFq7ho3HQ6tm7OY1cc60RjVsfUZLK5qIrlxwBfk7QIGJysI6kwGakAoDcwXdJc4DlSPdDmSyog1RTXB5gtaY6kkckxfwfWSHqDVCeCn0TEmmrVzLLqz3OXc+mEIg7q2JpHLx/EAe1aZjskM6siRezZowpJG/nsc5kNpIas+Y+0ZyZ1UmFhYRQVefSdXPPg9Pe4ZvJ8ju6ez9hhhezjicXMcoqkWRFRWFm5qjyzuYXU85AHSTWZnUfqpcrZwD182o3ZrEb87tl/c8PfFvDlQzty54VH0bJZ42yHZGZ7qSrJ5rSIODxt/S5JcyLip5J+XtOBWcMVEdzwt4X8/rl/c9rh+/Prcw6nqeeOMavTqvJ/cLGkcyQ1Sj7nAFuSfe42bDWipDT4+eOv8fvn/s2FA7tx87lHONGY1QNVubO5ELiV1EjPAbwCDEl6i303A7FZA7NtRyk/enQOf5m3gu98+SB+/PVDPXKzWT1RlVGfF5N6U78iL9RMONZQbd5WwhUPzOLZhav5+cm9GPXFg7IdkpnVoD1un5B0iKSpyYuVSOon6drMhWYNxYbN27lo3HSef2s1Y87s60RjVg9VpTH8blKjKW8HiIh5pHqkme211Ru3cv5drzB32Xpuv+BIzhvQLdshmVkGVOWZTV5EzCjXhr6jhuOxBmTZumIuGjeDDzZsYezQo/nSIR2zHZKZZUhVks2Hkg4i6Xkm6duk5psxq7K3V33MReOms2nrDu4fOYCjuudnOyQzy6CqJJvvkBr4spek94ElpHqomVXJ/GUbGDp+Bo0kHrlsEL277JPtkMwsw6qSbN4HxpMabywf+IjUAJq/ykBcVk+9sngNIycU0bZlUx4YOZAeHVplOyQzqwVVSTZPkJqOeTawvJKyZp8x9c2VXPnAbLrm53H/iIF0blvV+fbMrK6qSrIpiIhdznpptjtPzHmf/3h0Ln3234d7hw8gv1WzbIdkZrWoKl2fX5LUN2ORWL018eV3uOqRORT2aM8DIwc60Zg1QFW5szkeGCZpCbCV1MjPERH9MhKZ1XkRwR3T3uamf7zF4N6duP2C/rRo6pGbzRqiqiSbk2ryiyXlA48APYB3gHMiYl25Mt2Bx0ndgTUFfhsRv5eUBzxGaoqDEuDPETE6OeZm4MvJKfKA/SKiXU3GbpWLCP7nqTe5+19LOLP/Adz47X408YCaZg1WVcZGe7eGv3s0MDUixkganaz/tFyZFcCgiNgqqTXwmqQnSXVUuCkipklqBkyVdFJE/DUiflh2sKTvAf1rOG6rxI6SUn7++HweLVrGsGN78Itv9qFRIw+oadaQZfNPzdOBCcnyBOCM8gUiYltEbE1Wm5PEGxHFETGtrAypHnIFFXzH+cBDNRy37cbWHSV898FXebRoGT/4ak/+81QnGjPLbrLpFBFlIxB8AHSqqJCkrpLmAUuBGyJiebn97UiNRj213PbuwIHAP2s6cKvYpq07GDmhiL+9/gG/+GYffvi1QzxFgJkBVXtmU2WSngE6V7DrmvSViAhJFU7AFhFLgX6S9gcmS5oUESuT8zchdedyWzIFQrrzgEkRUbKb+EYBowC6dfMAkNWxvngbw++dydyl6/m/b/fj7MKu2Q7JzHJIRpNNRAze1T5JKyV1iYgVkroAqyo51/JkeoMvAJOSzXcBiyLilgoOOY/UEDu7O+ddyTkoLCz0bKN7adVHW7ho3AyWfLiJOy88ihMPq+jvCzNryLLZjPYkqeFuSH4+Ub6ApIJkJlAktSfV/Xphsn4d0Ba4qoLjegHtgZczErl9YunaYs7+w8ssXVfM+OFHO9GYWYWymWzGAF+TtAgYnKwjqVDS2KRMb2C6pLnAc6R6oM2XVECqKa4PMFvSHEkj0859HvBwRPhuJYPeWrmRs373EuuLt/PAyIEcd3CHbIdkZjlK/n2cUlhYGEVFRdkOo86Ys3Q9w8bPoFnjRkwcMZBDO7fJdkhmlgWSZkVEYWXlMvrMxuqnl97+kEvvK2Lf1s25f8RAuu2bl+2QzCzHOdlYlfz99Q/43oOvcmCHVkwcMYD99vHIzWZWOScb22N/nLWMq/84j74HtOXe4UfTLs8DaprZnnGysT1yzwtL+NVf3uD4gzvwh4uOolVz/6djZnvOvzFstyKCW55ZxK1TF3Hi5ztz6/lH0LyJR242s6pxsrFdKi0NfvWXN7j3pXf49lEFjDmzr0duNrO94mRjFdpRUsrVf5zHn2a/z4jjD+Sak3t7QE0z22tONvYZW7aX8L2HXuXpN1byH187hO9+5WAPqGlm1eJkYzv5eOsOLp1QxMuL1/Bfp32eocf2yHZIZlYPONnYJ9Zt2saw8TN4bflH3HLuEZzR/4Bsh2Rm9YSTjQHwwYYtXDRuOu+uLeYPQ45icJ8KpxcyM9srTjbGOx9uYsi46awv3s6E4QMYdNC+2Q7JzOoZJ5sG7s0VH3HRuBmUlJby4KUD6VfQLtshmVk95GTTgM16dx3Dx88gr1kTHh41iIP388jNZpYZTjYN1PNvreayibPotE9z7h85kIL2HrnZzDLHyaYBemr+Cn7w8KscvF8b7rtkAB3bNM92SGZWz2Vt7BFJ+ZKelrQo+dm+gjLdJZXNxPm6pMuT7XmSpkhakGwfk3ZMN0nTJL0qaZ6kk2uzXrnu0ZlL+e6Ds+lX0I6HRx3jRGNmtSKbA12NBqZGRE9garJe3gpgUEQcAQwERkvaP9l3U0T0AvoDx0k6Kdl+LfBoRPQnNT30nZmsRF1y9/OLufqP8/hCz45MHDGAti2bZjskM2sgsplsTgcmJMsTgDPKF4iIbRGxNVltThJvRBRHxLSyMsBsoKDsMGCfZLktsDwj0dchEcH//X0B1z/1Jqf07cLdFxeS18wtqGZWe7L5G6dTRKxIlj8AKnyLUFJXYApwMPCTiFhebn874FTg1mTTL4F/SPoe0AoYXPOh1x2lpcF/Pvk6E1+EoX10AAAOyUlEQVR5l/MHdOW6M/rS2ANqmlkty2iykfQM0LmCXdekr0RESIqKzhERS4F+SfPZZEmTImJlcv4mwEPAbRGxODnkfODeiPi1pEHAREmHRURpBfGNAkYBdOvWbe8qmcO2l5Ty48fm8sSc5Vz2pc8x+sReHlDTzLIio8kmInZ5VyFppaQuEbFCUhdgVSXnWi7pNeALwKRk813Aooi4Ja3oCODE5JiXJbUAOlR0/oi4KzkHhYWFFSa7umrL9hKufGA2/1ywiqtPPJQrTzg42yGZWQOWzWc2TwJDk+WhwBPlC0gqkNQyWW4PHA8sTNavI/VM5qpyh70HfDUp0xtoAazOQPw566Mt27n4nhlMW7iK6791mBONmWVdNpPNGOBrkhaReq4yBkBSoaSxSZnewHRJc4HnSPVAmy+pgFRTXB+grGv0yOSY/wAuTY55CBgWEfXqrmV31ny8lQvufoXZ767j1vP6c+HA7tkOycwMNaDfw7tVWFgYRUVF2Q6jWpav38yQcdN5f91mfj/kKL7ca79sh2Rm9ZykWRFRWFk593+tJxav/pghY6ezccsOJo4YyIAD87MdkpnZJ5xs6oHX3t/A0HtmAPDQqGM47IC2WY7IzGxnTjZ13Mx31nLJ+Jm0adGE+0cO5HMdW2c7JDOzz3CyqcOmLVjFFQ/MYv92Lbl/xED2b9cy2yGZmVXIyaaO+vPc5fzwkTkc2rkNEy4ZQIfWHlDTzHKXk00d9MD0d7l28msc3SOfsUML2aeFB9Q0s9zmZFPH3Pns29z4t4V8pdd+3HnhkbRo2jjbIZmZVcrJpo6ICMb8bQF/eG4xpx2+P78+53CaNs7mO7lmZnvOyaYOKCkNrp08n4dmLGXIMd341WmH0cgjN5tZHeJkk+O27Sjlh4/OYcq8FXznywfx468f6pGbzazOcbLJYZu3lXD5/bN47q3V/PzkXoz64kHZDsnMbK842eSoDZu3M+Lemcx+bx03nNWXc4+uf/PtmFnD4WSTg1Zv3MrF98zg7VUbuf2CIzm5b5dsh2RmVi1ONjlm2bpihoydzsqPtjJu6NF88ZCO2Q7JzKzanGxyyNurNjJk7AyKt+3g/pEDOKq7R242s/rBySZHzFu2nqH3zKBxo0Y8ctkgenfZJ9shmZnVmKy9FSgpX9LTkhYlP9tXUKa7pLKZOF+XdHmyPU/SFEkLku1jyh0zVdI8Sc8ms3rmtJf/vYYL7p5Oq+ZNmHS5E42Z1T/ZfAV9NDA1InoCU5P18lYAgyLiCGAgMFrS/sm+myKiF9AfOE7SSWXbgfsioh/wK+B/M1mJ6nrmjZUMHT+Dzm1bMOnyY+nRoVW2QzIzq3HZTDanAxOS5QnAGeULRMS2iNiarDYniTciiiNiWlkZYDZQdgfTB/hnsjwt+Z6cNPnV97ns/ln07tyGRy8bROe2LbIdkplZRmQz2XSKiBXJ8gdAp4oKSeoqaR6wFLghIpaX298OOJXU3RHAXODMZPlbQBtJ+9Z08NV138vvcNUjcxjQI58HLj2G/FbNsh2SmVnGZLSDgKRngM4V7LomfSUiQlJUdI6IWAr0S5rPJkuaFBErk/M3AR4CbouIxckhPwZulzQMeB54HyjZRXyjgFEA3brVzkuTEcHt/3ybXz/9FoN7d+L2C/p75GYzq/cUUeHv+Mx/sbQQOCEiVkjqAjwbEYdWcsw9wFMRMSlt/eOI+P4uyrcGFkREpZ0ECgsLo6ioqMr1qIqI4PopbzL2hSWc2f8Abvx2P5p45GYzq8MkzYqIwsrKZfM33ZPA0GR5KPBE+QKSCiS1TJbbA8cDC5P164C2wFXljukgqaxePwPuyUj0VbSjpJSrJ81j7AtLGHZsD246+3AnGjNrMLL5224M8DVJi4DByTqSCiWNTcr0BqZLmgs8R6oH2vykO/M1pDoDlHWNHpkccwKwUNJbpJ4DXV9rNdqFrTtK+O6Dr/LYrGX84Ks9+c9T+3iKADNrULLWjJZrMtWMtmnrDi6bOIsX3v6QX3yzD5ccf2CNf4eZWbbsaTOaRxDIoPXF2xg2fibzlq3nprMP59tH5fz7pWZmGeFkkyGrPtrCReNmsOTDTfxuyFF84/MVdcozM2sYnGwy4L01xQwZN50PP97K+OFHc9zBHbIdkplZVjnZ1LC3Vm5kyNjpbN1RygMjB9K/22eGfDMza3CcbGrQq++tY/i9M2nWuBGPXjaIQzu3yXZIZmY5wcmmhrz49odcel8RHVo35/4RA+m2b162QzIzyxlONjXgb699wPcfepUDO7Ri4ogB7LePB9Q0M0vnZFNNk2Yt4+pJczm8azvGDzuadnkeUNPMrDwnm2qKCI47uAO/H3IUrZr7n9PMrCL+7VhNZxd25awjCzz8jJnZbngkyBrgRGNmtntONmZmlnFONmZmlnFONmZmlnFONmZmlnFONmZmlnFONmZmlnFONmZmlnGeFjohaTXw7l4e3gH4sAbDySbXJffUl3qA65KrqlOX7hHRsbJCTjY1QFLRnszBXRe4LrmnvtQDXJdcVRt1cTOamZllnJONmZllnJNNzbgr2wHUINcl99SXeoDrkqsyXhc/szEzs4zznY2ZmWWck00lJJ0oaaGktyWNrmD/MEmrJc1JPiPT9g2VtCj5DK3dyD8TZ3XqUZK2/cnajfyzKqtLUuYcSW9Iel3Sg2nbc+aaJPFUpy516rpIujkt3rckrU/blzPXpZr1qGvXpJukaZJelTRP0slp+36WHLdQ0jeqHUxE+LOLD9AY+DfwOaAZMBfoU67MMOD2Co7NBxYnP9sny+3rWj2SfR9n+1pUsS49gVfL/r2B/XLtmlS3LnXxupQr/z3gnly7LtWpR128JqSe1VyRLPcB3klbngs0Bw5MztO4OvH4zmb3BgBvR8TiiNgGPAycvofHfgN4OiLWRsQ64GngxAzFWZnq1CPX7EldLgXuSP7diYhVyfZcuiZQvbrkmqr+N3Y+8FCynEvXpTr1yDV7UpcA9kmW2wLLk+XTgYcjYmtELAHeTs6315xsdu8AYGna+rJkW3lnJbegkyR1reKxtaE69QBoIalI0iuSzshopJXbk7ocAhwi6cUk5hOrcGxtqk5doO5dFwAkdSf11/I/q3psLahOPaDuXZNfAkMkLQOeInWntqfHVomTTfX9GegREf1I/UU2Icvx7K3d1aN7pN4uvgC4RdJB2QiwCpqQan46gdRfnndLapfViPbe7upS165LmfOASRFRku1AqqmietS1a3I+cG9EFAAnAxMlZSQvONns3vtA+l/4Bcm2T0TEmojYmqyOBY7a02NrUXXqQUS8n/xcDDwL9M9ksJXYk3/XZcCTEbE9aQJ4i9Qv7Fy6JlC9utTF61LmPHZuesql61KdetTFazICeBQgIl4GWpAaJ63mr0m2H2Ll8ofUX5WLSd0qlz1g+3y5Ml3Slr8FvJIs5wNLSD3wbJ8s59fBerQHmifLHYBF7OaBaY7U5URgQlrMS4F9c+ma1EBd6tx1Scr1At4heccv2ZYz16Wa9ahz1wT4KzAsWe5N6pmNgM+zcweBxVSzg0BW/hHq0ofUreVbpHpjXJNs+xVwWrL8v8DryYWZBvRKO/YSUg/W3gaG18V6AMcC85Pt84ERdeCaCPgN8EYS83m5eE2qU5e6eF2S9V8CYyo4Nmeuy97Woy5eE1K9zl5MYp4DfD3t2GuS4xYCJ1U3Fo8gYGZmGednNmZmlnFONmZmlnFONmZmlnFONmZmlnFONmZmlnFONmZmlnFONmaVkPRLST/OwvcOk3T7XhxXpXglHZE+tLxZJjjZmNkRpF7++wxJTWo5FqunnGyswZJ0cTLK9VxJEyX1kPTPZNtUSd0qOOb7yURm8yQ9nGwbIOnlZAKqlyQdmmwfJmmypKclvSPpu5J+lJR7RVJ+Uu5ZSbcmE269JukzQ7lL6ijpj5JmJp/jKqne4UlMiyRdmpzjvvSRiCU9IOl0Um+Un5t8/7nJndFESS+SGpixsaT/S753nqTL0s7xk7Tt/1X1q2ANhf9qsQZJ0ueBa4FjI+LD5Bf/BFLjkE2QdAlwG1B+mPjRwIERsTVt9OUFwBciYoekwcD/AGcl+w4jNRhjC1JDsfw0IvpLuhm4GLglKZcXEUdI+iJwT3JculuBmyPihSQJ/p3UWFa70g84BmgFvCppCjAO+CEwWVJbUsOrDCU1pldhRHw3+bf5JalhTI6PiM2SRgEbIuJoSc2BFyX9g9SAoD1JzXMi4ElJX4yI53cTlzVQTjbWUH0FeCwiPgSIiLWSBgFnJvsnAjdWcNw84AFJk4HJyba2wARJPUlNRtU0rfy0iNgIbJS0gdRUDpAaO6tfWrmHkjiel7RPBVMiDAb6SCpb30dS64j4eBf1eyIiNgObJU0DBkTEZEl3SupIKhn+MUmQFR3/ZHI8wNeBfpK+nVbfnsn2r5OaSRSgdbLdycY+w8nGrGpOAb4InApcI6kv8N+kksq3JPUgNbR8ma1py6Vp66Xs/P9f+UEKy683Ao6JiC17GOeuzncfMITU8PjDd3P8prRlAd+LiL+nF1BqXvr/jYg/7GFM1oD5mY01VP8Ezpa0L0DSjPYSqV/CABcC/0o/IJlUqmtETAN+Suov/NbJz7K5PobtZTznJt9xPKkmqw3l9v+DT2dRRNIRlZzvdEktkvqdAMxMtt8LXAUQEW8k2zYCbXZzrr8DV0hqmnz3IZJaJdsvkdQ62X6ApP0qicsaKN/ZWIMUEa9Luh54TlIJqaag7wHjJf0EWM1n//JvDNyfPO8QcFtErJd0I6lmtGuBKXsZ0hZJr5Jqgrukgv3fB+6QNI/U/7fPA5fv5nzzSE0V0QH474hYDhARKyW9yadNgCTlRkuaQ2qqifLGAj2A2Uq1ua0GzoiIf0jqDbycNMV9TOquadWeVdkaEk8xYJZlkp4FfhwRRbXwXXmknhcdWcHdk1nGuBnNrIFIesq9CfzWicZqm+9szOooScOBH5Tb/GJEfCcb8ZjtjpONmZllnJvRzMws45xszMws45xszMws45xszMws45xszMws4/4/hHUaEQD3Pp4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "plt.plot(colsample_bytree_s, test_means)\n",
    "plt.xlabel( 'colsample_bytree' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最优值落在边界上，暂时不再继续调，colsample_bytree=0.8，待确定其他参数后再收缩范围查找"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "5)subsample\n",
    "采样比例，取值为(0, 1]，注意这里的子采样和随机森林不一样，随机森林使用的是放回抽样，而这里是不放回抽样。\n",
    "如果取值为1，则全部样本都使用，等于没有使用子采样。如果取值小于1，则只有一部分样本会去做 GBDT 的决策树拟合。\n",
    "选择小于1的比例可以减少方差，即防止过拟合，但是会增加样本拟合的偏差，因此取值不能太低，一般在[0.5, 0.8]之间。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   43.7s\n",
      "[Parallel(n_jobs=5)]: Done  16 out of  20 | elapsed:  1.4min remaining:   21.3s\n",
      "[Parallel(n_jobs=5)]: Done  20 out of  20 | elapsed:  1.4min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.8, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "        ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'subsample': [0.5, 0.6, 0.7, 0.8]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'num_leaves':70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':0.8\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "subsample_s = [i/10.0 for i in range(5,9)]\n",
    "tuned_parameters = dict( subsample = subsample_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.3215615820552076\n",
      "{'subsample': 0.8}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAAERCAYAAAAT5VIbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xu8lWWd9/HPV86gnOQgAopy0NIUYQ9iqZEioDXhdLAaCyyTtMxKm+exVzNjj/bM4zSjJtloaCbUdDBnGpnSwQ1qWom6KcVDujcgCFvkIMhB5Px7/ljXrsVurX3e+9577e/79bpf+76vdR1+11q++Hnd6173rYjAzMysozss6wDMzMxaghOamZmVBCc0MzMrCU5oZmZWEpzQzMysJDihmZlZSXBCa0ck3SBpuaRnJD0k6egCdcZLekLSC6nux/Jeu1LSCkkhaVBe+RRJ21K/z0j6x7zX+ku6T9JLkv4o6YwWmMf/SHpT0i+b25eZWUPJv0NrPyT1jYjtaf8q4J0RcXmtOuOAiIiqlPCWAe+IiDclnQZsBR4FyiJic2ozBfhqRHygwJjzgccj4i5J3YHeEfFmM+dxLtAb+FyhMc3MWoNXaO1ITTJL+gB/8X8bEVEZEVVp/zVgIzA4Hf8hIlY3dDxJ/YCzge+n9ntrkpmk0WmltUzS45JObMQ8lgA7GlrfzKwlOKG1M5L+r6S1wMXAP9ZTdxLQHVjZgK7PkPSspAclnZTKjgM2AT+Q9AdJd0nqk16bB3wxIiYCXwX+rSnzMTNrKz7l2MYkLQaOKvDS1yPi/rx6XwN6RsR1RfoZRu7U4uyIWFrrtdUcesqxL3AwInZKugC4NSLGSioDlgLviYgnJd0KbAf+mVyiezmv2x4R8Q5JHwKuLxBSdURMz4thCkVOc5qZtQYntHZK0jHAAxFxcoHX+pJLZv8UEfcVeH01eQmt2OtAV2BpRIxK5WcB1wKfAF6OiGHNiH8KTmhm1oZ8yrEdkTQ273Am8FKBOt2BXwALCiWzIv0eJUlpfxK5z/2NiHgdWCvphFT1XODF9F3eK5I+mtpI0qlNnZeZWVtwQmtfbpT0vKTlwDTgSwCSyiTdlepcRO5CjkvyLsMfn+pdJWkdMAJYntfmI8Dzkp4F5gIfjz8vzb8I/HsaczzwT6n8YuDS1OYFcgm2QSQ9DvwcOFfSOknT62tjZtZcPuVoZmYlwSs0MzMrCV2zDqAzGTRoUIwaNSrrMMzMOpRly5ZtjojB9dVzQmtDo0aNoqKiIuswzMw6FElrGlLPpxzNzKwkZJbQJA2UVC6pKv0dUKTe7FSnStLsvPKJkp5LN+Odm3dZesF+06Xnc1P95ZImNGCM/ytpraSdtWK6RNKmvKsMP9vS74+ZmTVOliu0a4ElETEWWJKODyFpIHAdcDowCbguL/HdDlwGjE3bjHr6PT+v7pzUvr4x/juVFfKziBiftruK1DEzszaSZUKbCcxP+/OBCwvUmQ6UR8SWiNgKlAMz0m2f+kbE0vR7qgV57Yv1O5Pcj5Ej3Sqqf+qn4BgAqf/1LThnMzNrJVkmtKF5yeJ1YGiBOsOBtXnH61LZ8LRfu7yufuvqq1B5fT6cTl3eJ2lksUqS5kiqkFSxadOmBnRrZmZN0aoJTdLidOeL2tshd51Iq6wW/4V3a/VL7lTkqIg4hdyKbn6xihExLyLKIqJs8OB6rzo1M7MmatXL9iNiarHXJG2QNCwi1qdTfxsLVKsGpuQdjyB3U97qtJ9fXp32i/VbDYws0KbYGHXN6428w7uAb9VV38zMWl+WpxwXAjVXFM4G7i9QZxEwTdKAdKHGNGBROqW4XdLkdHXjrLz2xfpdCMxKVztOBralfgqOUVfgKVHW+CDwxwbP2sysE9mz/wDfWPgCr76xq9XHyjKh3QicJ6kKmJqOD7kRb0RsAW4Ank7b9akM4PPkVkcryD3g8sG6+gUeAFal+nem9nWOIelb6Wa/vdNNdr+R+rpK0gvpxr1XAZe04PtiZlYy7n16Lff8bjWr33ir1cfyzYnbUFlZWfhOIWbWWezed4D3/ssjHDOwN/d+7gzSz4UbTdKyiCirr57vFGJmZq3iR0vXsGH7Hq4+74QmJ7PGcEIzM7MW99ae/dzx65W8Z8yRnDH6yDYZ0zcnNjOzFjf/idVs3rmX7513QpuN6RWamZm1qB279zHvsVW874TBTDy24G16W4UTmpmZtai7f7OaN3ft4+o2XJ2BE5qZmbWgN3ft5a7HVzHtnUN514h+bTq2E5qZmbWYOx9fxc69+7l62rg2H9sJzczMWsQbO/fwg9+u5v3vGsaJR/Vt8/Gd0MzMrEXc8euV7N53gC9PbfvVGTihmZlZC9i4fTcLnljDhacNZ8yQwzOJwQnNzMya7buPrGD/weBL547NLAYnNDMza5bqN9/mJ0+t5aKyERx7ZJ/M4nBCMzOzZrnt4SoArjwnu9UZOKGZmVkzvPrGLn5esY5PTBrJ8P69Mo3FCc3MzJrs1iVVdDlMfOF9Y7IOxQnNzMyaZsXGnfziD+v41ORjGdK3Z9bhOKGZmVnT3Lqkip7dunD5lNFZhwJkmNAkDZRULqkq/S14S2ZJs1OdKkmz88onSnpO0gpJc5WeHlesX+XMTfWXS5pQ1xiSekv6laSXJL0g6ca8+j0k/Sz19aSkUa3zLpmZtU8vvb6dXy5/jUvePYpBh/fIOhwg2xXatcCSiBgLLEnHh5A0ELgOOB2YBFyXl/huBy4DxqZtRj39np9Xd05qX98Y/xoRJwKnAe+RdH4qvxTYGhFjgFuAf27eW2Fm1rHcUl7J4d27Mufs47MO5U+yTGgzgflpfz5wYYE604HyiNgSEVuBcmCGpGFA34hYGhEBLMhrX6zfmcCCyFkK9E/9FBwjInZFxCMAEbEX+D0wosAY9wHn1qwQzcxK3XPrtrHohQ1cetZx9O/dPetw/iTLhDY0Itan/deBoQXqDAfW5h2vS2XD037t8rr6rauvQuV/Iqk/8NfkVnyH9BUR+4FtQMFnjEuaI6lCUsWmTZsKVTEz61BuLn+Zfr268Zkzj8s6lEN0bc3OJS0Gjirw0tfzDyIiJEVLj98S/UrqCvwEmBsRq5oQwzxgHkBZWVmLz9HMrC0tW7OVR17exP+acQJ9e3bLOpxDtGpCi4ipxV6TtEHSsIhYn079bSxQrRqYknc8Ang0lY+oVV6d9ov1Ww2MLNCm2Bg15gFVEfHtWnGNBNalhNcPeKPYXM3MSsUt5ZUc2ac7s88YlXUofyHLU44LgZqrFmcD9xeoswiYJmlAulBjGrAonVLcLmly+u5qVl77Yv0uBGalqx0nA9tSPwXHAJD0TXLJ6st1xP4R4OH0XZ6ZWclauuoNfrNiM1dMGU2fHq26HmqSLCO6EbhX0qXAGuAiAEllwOUR8dmI2CLpBuDp1Ob6iNiS9j8P3AP0Ah5MW9F+gQeAC4AVwC7g0wDFxpA0gtyp0ZeA36drPm6LiLuA7wM/lLQC2AJ8vOXeFjOz9iciuPmhSoYc0YNPTj4263AKkhcWbaesrCwqKiqyDsPMrNEeq9zErLuf4vqZJzGrjU83SloWEWX11fOdQszMrE4RwU3llQzv34uP/dXI+htkxAnNzMzq9PBLG3l27Zt88Zwx9OjaJetwinJCMzOzog4eDG56qJJjBvbmwxNH1N8gQ05oZmZW1KIXXufF9dv58tSxdOvSvlNG+47OzMwyc+BgcMviSkYP7sPM8cPrb5AxJzQzMyvol8tfo3LDTr5y3ji6HNb+b1frhGZmZn9h/4GDfHtxFScedQQXnDws63AaxAnNzMz+wn/+oZpXNr/FV84bx2EdYHUGTmhmZlbL3v0HmbukincN78e0dxZ6EEr75IRmZmaH+Pmytazb+jZXTxtHR3rUoxOamZn9ye59B/jOkhVMPHYAU8YNzjqcRnFCMzOzP/nJU6/y+vbdXHNex1qdgROamZklb+89wHcfWcnk4wfy7jGDsg6n0ZzQzMwMgAVPrGbzzj1cM+2ErENpEic0MzNj55793PHrlZw9bjB/NWpg1uE0iROamZnxg9+8wtZd+7j6vHFZh9JkTmhmZp3ctl37mPf4Kqa+YyjjR/bPOpwmyyyhSRooqVxSVfo7oEi92alOlaTZeeUTJT0naYWkuUqX4xTrVzlzU/3lkibUNYak3pJ+JeklSS9IujGv/iWSNkl6Jm2fba33ycystd31m1Xs2L2/Q6/OINsV2rXAkogYCyxJx4eQNBC4DjgdmARcl5f4bgcuA8ambUY9/Z6fV3dOal/fGP8aEScCpwHvkXR+Xng/i4jxaburWe+EmVlGtry1l7t/8wrvf9cw3nl036zDaZYsE9pMYH7anw9cWKDOdKA8IrZExFagHJghaRjQNyKWRkQAC/LaF+t3JrAgcpYC/VM/BceIiF0R8QhAROwFfg+076fbmZk10vceW8mufQf48tSxWYfSbFkmtKERsT7tvw4UumHYcGBt3vG6VDY87dcur6vfuvoqVP4nkvoDf01uxVfjw+nU5X2SRhabpKQ5kiokVWzatKlYNTOzNrdxx27m/241M089mrFDj8g6nGZr1YQmabGk5wtsM/PrpVVWtPT4LdGvpK7AT4C5EbEqFf83MCoiTiG3optfrH1EzIuIsogoGzy4Y91GxsxK2+2PrmTfgeBLUzv2d2c1urZm5xExtdhrkjZIGhYR69Opv40FqlUDU/KORwCPpvIRtcqr036xfquBkQXaFBujxjygKiK+nTevN/Jevwv4VrF5mpm1R+u3vc2/P/kqH54wnOMG9ck6nBaR5SnHhUDNVYuzgfsL1FkETJM0IF2oMQ1YlE4pbpc0OV3dOCuvfbF+FwKz0tWOk4FtqZ+CYwBI+ibQD/hyflApUdb4IPDHJr0DZmYZue3hFUQEXzyn4393VqNVV2j1uBG4V9KlwBrgIgBJZcDlEfHZiNgi6Qbg6dTm+ojYkvY/D9wD9AIeTFvRfoEHgAuAFcAu4NMAxcaQNAL4OvAS8Pv0q4Db0hWNV0n6ILAf2AJc0mLviplZK1u7ZRf3VqzlY381kpEDe2cdTotR7msmawtlZWVRUVGRdRhm1sn93c+f5f5nX+Oxv3sfR/XrmXU49ZK0LCLK6qvnO4WYmXUir2x+i//8QzWfPP3YDpHMGsMJzcysE7l1cSXduxzGFVNGZx1Ki3NCMzPrJCo37OD+Z19j9rtHMfiIHlmH0+Kc0MzMOolvL66kT/eufO7s47MOpVU4oZmZdQIvvLaNB557nc+8ZxQD+nTPOpxW4YRmZtYJ3FJeSd+eXbn0rNJcnYETmplZyXtm7Zss/uNG5px9PP16dcs6nFbjhGZmVuJueuhlBvTuxiXvOS7rUFqVE5qZWQl76pUtPF61mSumjObwHlneHKr1OaGZmZWoiOCmh15m8BE9+NTkUVmH0+qc0MzMStTvVr7Bk69s4QtTRtOre5esw2l1TmhmZiWoZnU2rF9PPj7pmKzDaRNOaGZmJejRlzfx+1ff5MpzxtCzW+mvzsAJzcys5EQEN5dXMnJgLz46cWT9DUqEE5qZWYl56MUNPFe9javOGUv3rp3nn/nOM1Mzs07g4MHg5ocqOX5QH/7mtOFZh9OmnNDMzErIr55bz8sbdvClqWPp2qVz/RPf4NlK+pakvpK6SVoiaZOkTzZ1YEkDJZVLqkp/BxSpNzvVqZI0O698oqTnJK2QNFeS6upXOXNT/eWSJjRgjP+R9KykFyTdIalLY2I3M2tL+w8c5JbFlYwbejh/fcrRWYfT5hqTvqdFxHbgA8BqYAzwd80Y+1pgSUSMBZak40NIGghcB5wOTAKuy0setwOXAWPTNqOefs/Pqzsnta9vjIsi4lTgZGAw8NGGxm5m1tbuf+Y1Vm16i6vPG8dhhynrcNpcYxJazT1T3g/8PCK2NXPsmcD8tD8fuLBAnelAeURsiYitQDkwQ9IwoG9ELI2IABbktS/W70xgQeQsBfqnfgqOAZASeM3cuwPRiNjNzNrMvgMHuXVJFScd3ZfpJx2VdTiZaExC+6Wkl4CJwBJJg4HdzRh7aESsT/uvA0ML1BkOrM07XpfKhqf92uV19VtXX4XKAZC0CNgI7ADua0TsNe3nSKqQVLFp06Zi1czMmuW+Zet4dcsurpk2jvQNTKfT4IQWEdcC7wbKImIf8Ba5lUpRkhZLer7Adki7tMqKIt00WUv0GxHTgWFAD+Ccxo4REfMioiwiygYPHtycUMzMCtqz/wDfWVLF+JH9ed8JQ7IOJzONuSjko8C+iDgg6e+BHwF1fusYEVMj4uQC2/3AhnTKj/R3Y4EuqoH8XwWOSGXVab92OXX0W1dfhcrz57EbuJ8/J/CGxG5m1iZ++tRaXtu2u1OvzqBxpxz/ISJ2SDoTmAp8n3RhRRMtBGquKJxNLmHUtgiYJmlAulBjGrAone7bLmlyurpxVl77Yv0uBGalqx0nA9tSPwXHkHR4XtLqSu67w5caEbuZWavbve8A331kBZOOG8iZYwZlHU6mGpPQDqS/7wfmRcSvyF0o0VQ3AudJqiKXIG8EkFQm6S6AiNgC3AA8nbbrUxnA54G7gBXASuDBuvoFHgBWpfp3pvZ1jdEHWChpOfAMuVXYHfWMYWbWpn60dA0bd+zhmvM69+oMQLmvgBpQUfoluVNx5wETgLeBp9Jl7dYAZWVlUVFRkXUYZlYi3tqzn7O+9QgnHd2XH156etbhtBpJyyKirL56jVmhXUTu9Nz0iHgTGEjzfodmZmbNcM/vVrPlrb1cfd64rENpFxpzleMucqf2pku6EhgSEQ+1WmRmZlbU9t37mPfYKs45cQinHeObFUHjrnL8EvDvwJC0/UjSF1srMDMzK+77j7/Ctrf3eXWWp2v9Vf7kUuD0iHgLQNI/A08A32mNwMzMrLCtb+3l7t+8woyTjuLk4f2yDqfdaMx3aOLPVzqS9jv3JTVmZhmY9/gqdu7dz1e8OjtEY1ZoPwCelPSLdHwhud+imZlZG9m8cw/3/HY1f33K0Zxw1BFZh9OuNDihRcTNkh4FzkxFn46IP7RKVGZmVtAdj65kz/4DfGnq2KxDaXfqTWjp8So1VqftT6/l/dDZzMxa0Ybtu/nh0jV8aMIIRg8+POtw2p2GrNCWkbv5bs33ZTW/xFbaP74V4jIzs1q++8gKDhwMvnSuV2eF1JvQIuK4hnQk6aSIeKH5IZmZWW3rtu7iJ0+9ykfLRjJyYO+sw2mXGnOVY31+2IJ9mZlZntseXoEQXzxnTNahtFstmdB8Cb+ZWStYvfktfr5sHX97+jEc3b9X1uG0Wy2Z0Fr8AZ1mZgZzl1TRrYv4/JTRWYfSrrVkQjMzsxa2YuMO/uuZamadMYohfXtmHU671pIJbW8L9mVmZsAti6vo2a0LnzvbF5TXp8E/rJY0oUDxNmBNROyPiMktF5aZmf1x/XZ+tXw9V75vDEce3iPrcNq9xtz66t/IPdhzObkLQE4GXgD6SbrCj5IxM2tZt5RXckTPrlx2lldnDdGYU46vAadFRFlETAROA1aRe4L1txo7sKSBksolVaW/BR/oI2l2qlMlaXZe+URJz0laIWmu0rPHi/WrnLmp/vL8FWcdY/yPpGclvSDpDkldUvk3JFVLeiZtFzR2/mZmdVm+7k0eenEDl511PP16d8s6nA6hMQltXP4PpyPiReDEiFjVxLGvBZZExFhgSTo+RLrt1nXA6cAk4Lq8xHc7cBkwNm0z6un3/Ly6c1L7+sa4KCJOJbcaHQx8NC+8WyJifNoeaOJ7YGZW0M3llfTv3Y1Pv2dU1qF0GI1JaC9Iul3Se9P2b8CLknoA+5ow9kxgftqfT+7u/bVNB8ojYktEbAXKgRmShgF9I2JpRASwIK99sX5nAgsiZynQP/VTcAyAiNie2nYFuuOfJphZG1i2ZguPvryJz509miN6enXWUI1JaJcAK4Avp21VKtsHvK8JYw+NiPVp/3VgaIE6w4G1ecfrUtnwtF+7vK5+6+qrUDkAkhYBG4EdwH159a5Mpy7vLna61MysKW56qJJBh3dn9ruPzTqUDqXBCS0i3ib3dOp/BP4BuDUidkXEwYjYWaiNpMWSni+wzazVd9AKq5+W6DcipgPDgB7AOan4dmA0MB5YD9xUrL2kOZIqJFVs2rSpOaGYWSfwu5Wb+d3KN7hiyhh6d2/MdXvW4IQmaQpQBdxG7orHSkln19UmIqZGxMkFtvuBDemUH+nvxgJdVAMj845HpLLqtF+7nDr6rauvQuX589gN3E/utCURsSEiDkTEQeBOct+9FXsP5qULacoGDx5crJqZGRHBzQ9VclTfnlx8+jFZh9PhNOaU403AtIh4b0ScTe67p1uaMfZCoOaKwtnkEkZti4Bpkgak03rTgEXplOJ2SZPT1Y2z8toX63chMCtd7TgZ2Jb6KTiGpMPzEmNX4P3AS+l4WF6MfwM834z3wcwMgMeqNlOxZitfOGcMPbt1yTqcDqcx69luEfFyzUFEVEpqzreVNwL3SroUWANcBCCpDLg8Ij4bEVsk3QA8ndpcn/dA0c8D9wC9gAfTVrRf4AHgAnLfA+4CPp3mUXAMSUOBhemil8OAR4A7Up1vSRpP7nTmauBzzXgfzMzS6uxlhvfvxcfKRtbfwP6Ccl8zNaCidDdwEPhRKroY6BIRn2ml2EpOWVlZVFRUZB2GmbVD5S9u4LIFFXzrw6dw0V85oeWTtCwiyuqr15gV2hXAF4Cr0vHj5L5LMzOzZjh4MLi5vJJRR/bmQxOG19/ACmpwQouIPcDNaTMzsxby4POv88f127nlY6fStYsfgtJU9SY0Sc9Rx6XvEXFKi0ZkZtaJHDgY3LK4kjFDDueDp3p11hwNWaF9oNWjMDPrpP772ddYsXEn3/3bCXQ5TFmH06HVm9AiYk1DOpL0RESc0fyQzMw6h/0HDvLtxZW8Y1hfzj/5qKzD6fBa8mStH6VqZtYI//n7ala/sYurzxvHYV6dNVtLJjTfuNfMrIH27j/IrUuqOHVEP6a+Y0jW4ZQEX05jZpaBn1WspfrNt7l62gmkxzlaM7VkQvMnYmbWALv3HeC2h6soO3YAZ48dlHU4JaMlE9qnWrAvM7OS9e9PvsqG7Xu4xquzFtXgH1ZL2sFffk+2DagArokI36DXzKweu/bu5/ZHV/Du0Udyxugjsw6npDTm1lffJvfwyx+TO734cXLPBPs9cDcwpaWDMzMrNQueWMPmnXv53qfGZR1KyWnMKccPRsT3ImJHRGyPiHnA9Ij4GeAnNpuZ1WPH7n3c8euVTDlhMBOPHZh1OCWnMQltl6SLJB2WtouA3ek1X7JvZlaPH/x2NW/u2sfV53l11hoak9AuJnfhx0ZgQ9r/pKRewJWtEJuZWcnYtmsfdz6+imnvHMopI/pnHU5Jaszd9lcBf13k5d+0TDhmZqXpzsdXsWP3fr7i1VmrafAKTdI4SUskPZ+OT5H0960XmplZaXhj5x5+8NtXeP8pw3jHsL5Zh1OyGnPK8U7ga8A+gIhYTu5KRzMzq8P3HlvF2/sO8JWpY7MOpaQ1JqH1joinapXtb+rAkgZKKpdUlf4WvFJS0uxUp0rS7LzyiZKek7RC0lylXycW61c5c1P95ZIm1DdG3usLa1amjYndzGzj9t0seGI1F44fzpghR2QdTklrTELbLGk06YpGSR8B1jdj7GuBJRExFliSjg8haSBwHXA6MAm4Li953A5cBoxN24x6+j0/r+6c1L6+MZD0IWBnY2M3MwP4t0dXsu9AcNW5Xp21tsYktC8A3wNOlFQNfBm4vBljzwTmp/35wIUF6kwHyiNiS0RsBcqBGZKGAX0jYmlEBLAgr32xfmcCCyJnKdA/9VNwDABJhwNXA99sQuxm1sm99ubb/PjJV/noxBGMGtQn63BKXmPuFFIN/AB4BBgIbAdmA9c3ceyhEVGzwnsdGFqgznBgbd7xulQ2PO3XLq+r37r6KlQOcANwE7CrCbEDIGkOuRUhxxxzTLFqZlaCbntkBUFw5Tljsg6lU2hMQrsfeJPcra5ea0gDSYuBQo9h/Xr+QUSEpBb/cXZz+pU0HhgdEV+RNKqpY6Q7qswDKCsr8w/QzTqJV9/Yxb1Pr+VvTz+GEQN6Zx1Op9CYhDYiImbUX+3PImJqsdckbZA0LCLWp1N/GwtUq+bQe0SOAB5N5SNqlVen/WL9VgMjC7QpNsYZQJmk1eTepyGSHo2IKXWMYWYGwNyHq+hymPjC+7w6ayuN+Q7td5Le1YJjLyR3ypL09/4CdRYB0yQNSBdqTAMWpdN92yVNTlc3zsprX6zfhcCsdLXjZGBb6qfYGLdHxNERMQo4E6hMyayhsZtZJ7Vq007+8/fr+OTkYxnat2fW4XQajVmhnQlcIukVYA+5O+5HRJzSxLFvBO6VdCmwBrgIQFIZcHlEfDYitki6AXg6tbk+Irak/c8D9wC9gAfTVrRf4AHgAmAFue/EPk1uAnWN0ajYzcwAvr24ih5du3DFlNFZh9KpKHeRYAMqSscWKo+INS0aUQkrKyuLioqKrMMws1b08us7mHHrY1z+3tH87xknZh1OSZC0LCLK6qvXmHs5OnGZmdXjlvJKDu/elc+dfXzWoXQ6jfkOzczM6vB89Tb+54XX+cyZx9G/d/esw+l0nNDMzFrILeWV9OvVjUvPOi7rUDolJzQzsxbw+1e3suSljcw5+3j69uyWdTidkhOamVkLuKW8kiP7dOeSd4/KOpROywnNzKyZnlz1Bo9XbeaKKaPp06Mxv4ayluSEZmbWDBHBTeWVDDmiB5+cXPDXTdZGnNDMzJrhtyve4KlXtvCF942hZ7cuWYfTqTmhmZk1UUTwrw+9zNH9evLxSSPrb2CtygnNzKyJHnl5I8+sfZMvnjuWHl29OsuaE5qZWRNEBDc9VMkxA3vzkYkj6m9grc4JzcysCRa98DovvLadL507lm5d/E9pe+BPwcyskQ4cDG4ur+T4wX248LTh9TewNuGEZmbWSL9c/hqVG3bylanj6HKYsg7HEic0M7NG2H/gILcuruLEo47g/e8alnU4lscJzcysEf7rmddYtfktvjx1HId5ddauOKGZmTX+302JAAARGUlEQVTQvgMHuXVJJScP78v0k4ZmHY7VkllCkzRQUrmkqvR3QJF6s1OdKkmz88onSnpO0gpJcyWprn6VMzfVXy5pQn1j5L2+UNLzecffkFQt6Zm0XdCS742ZtU8/r1jH2i1vc815J5D+ybF2JMsV2rXAkogYCyxJx4eQNBC4DjgdmARcl5f4bgcuA8ambUY9/Z6fV3dOal/fGEj6ELCzQPy3RMT4tD3QpHfAzDqM3fsO8J2Hq5hwTH+mnDA463CsgCwT2kxgftqfD1xYoM50oDwitkTEVqAcmCFpGNA3IpZGRAAL8toX63cmsCBylgL9Uz8FxwCQdDhwNfDNFpu1mXVIP33qVdZv280107w6a6+yTGhDI2J92n8dKHRCejiwNu94XSobnvZrl9fVb119FSoHuAG4CdhVILYr06nLu4udLgWQNEdShaSKTZs2FatmZu3Y23sP8N1HV3L6cQN59+gjsw7HimjVhCZpsaTnC2wz8+ulVVa09PjN6VfSeGB0RPyiwMu3A6OB8cB6ckmvWAzzIqIsIsoGD/ZpCrOO6IdLV7Npxx6vztq5Vn0SXURMLfaapA2ShkXE+nTqb2OBatXAlLzjEcCjqXxErfLqtF+s32pgZIE2xcY4AyiTtJrc+zRE0qMRMSUiNuTN407gl8XmaWYd2849+7nj16s4a+wgJh03MOtwrA5ZnnJcCNRcUTgbuL9AnUXANEkD0mm9acCidEpxu6TJ6erGWXnti/W7EJiVrnacDGxL/RQb4/aIODoiRgFnApURMQUgJcoafwM8j5mVpHt++wpb3trLNdNOyDoUq0eWzwq/EbhX0qXAGuAiAEllwOUR8dmI2CLpBuDp1Ob6iNiS9j8P3AP0Ah5MW9F+gQeAC4AV5L4T+zRAPWMU8610SjKA1cDnGj99M2vvtr29j3mPrWLqO4YwfmT/rMOxeij3NZO1hbKysqioqMg6DDNroJvLK5m7pIpfXXUmJx3dL+twOi1JyyKirL56vlOImVkBW9/ay92/eYUL3nWUk1kH4YRmZlbA9x5bxVt79/PlqeOyDsUayAnNzKyWTTv2MP93q5l56tGMG3pE1uFYAzmhmZnVcvujK9l74CBf8uqsQ3FCMzPL8/q23fzoyTV86LThHDeoT9bhWCM4oZmZ5bntkSoigqvOHZt1KNZITmhmZsm6rbv42dNruahsJCMH9s46HGskJzQzs+Q7S1YgiSvPGZN1KNYETmhmZsDqzW9x3+/XcfHpxzCsX6+sw7EmcEIzMwNuXVJFty7iiimjsw7FmsgJzcw6vaoNO/ivZ6qZ/e5RDDmiZ9bhWBM5oZlZp/ftxVX07taFz53t1VlH5oRmZp3ai69t51fPreczZx7HwD7dsw7HmsEJzcw6tZvLK+nbsyufPev4rEOxZnJCM7NO69m1b7L4jxu47Kzj6derW9bhWDM5oZlZp3VTeSUDenfj02cel3Uo1gKc0MysU6pYvYXHKjdx+XtHc3iPrlmHYy0gs4QmaaCkcklV6e+AIvVmpzpVkmbnlU+U9JykFZLmSlJd/Spnbqq/XNKEBozxqKSXJT2TtiGpvIekn6W+npQ0qnXeJTNrLTc9VMmgw3sw64xRWYdiLSTLFdq1wJKIGAssSceHkDQQuA44HZgEXJeX+G4HLgPGpm1GPf2en1d3Tmpf3xgAF0fE+LRtTGWXAlsjYgxwC/DPzXkjzKxt/W7FZp5Y9QZfeN9oenXvknU41kKyTGgzgflpfz5wYYE604HyiNgSEVuBcmCGpGFA34hYGhEBLMhrX6zfmcCCyFkK9E/9FByjEbHfB5xbs0I0s/YtIripvJJh/XryiUnHZB2OtaAsE9rQiFif9l8HhhaoMxxYm3e8LpUNT/u1y+vqt66+CpXX+EE63fgPeUnrT20iYj+wDTiy0CQlzZFUIali06ZNhaqYWRv6deUmlq3ZypXnjKFnN6/OSkmrfhMqaTFwVIGXvp5/EBEhKVp6/Bbo9+KIqJZ0BPAfwKfIrQYbE8M8YB5AWVlZi8/RzBouIri5vJIRA3rx0Ykjsw7HWlirrtAiYmpEnFxgux/YkE75kf5uLNBFNZD/X92IVFad9muXU0e/dfVVqJyIqPm7A/gxue/YDulLUlegH/BG/e+ImWWp/MUNLF+3javOHUv3rr7Iu9Rk+YkuBGquKJwN3F+gziJgmqQB6UKNacCidEpxu6TJ6TTgrLz2xfpdCMxKVztOBralfgqOIamrpEEAkroBHwCeLzDGR4CH03d5ZtZOHTyYW50dN6gPHzpteP0NrMPJ8scXNwL3SroUWANcBCCpDLg8Ij4bEVsk3QA8ndpcHxFb0v7ngXuAXsCDaSvaL/AAcAGwAtgFfBqg2BiS+pBLbN2ALsBi4M5U5/vADyWtALYAH2+h98TMWskDz6/npdd3cOvHx9O1i1dnpUheWLSdsrKyqKioyDoMs07nwMFg2i2/psth4sEvnU2Xw3xRckciaVlElNVXz/+bYmYl7/5nqlm56S2+MnWck1kJc0Izs5K278BBbl1SxTuH9WX6SYUuurZS4YRmZiXtP5atY80bu7hm2jgO8+qspDmhmVnJ2rP/AN95eAXjR/bnnBOHZB2OtTInNDMrWfc+vZbqN9/mmmnj8N3pSp8TmpmVpN37cquzSaMGcuaYQVmHY23ACc3MStKPlq5h4449XO3VWafhhGZmJeetPfu549crOXPMICYfX/C+4VaCnNDMrOTMf2I1m3fu5epp47IOxdqQE5qZlZTtu/fxvV+v4n0nDGbCMQPqb2AlwwnNzErK3b95hW1v7+Pq807IOhRrY05oZlYy3ty1l+8//grTTxrKu0b0yzoca2NZ3m3fGuiOX6/kP5atq7+iWSe3c89+du7dz1fO83dnnZETWgcw6PAejB16eNZhmHUIE44ZwIlH9c06DMuAE1oH8JGJI/jIxBH1VzQz68T8HZqZmZUEJzQzMysJmSU0SQMllUuqSn8L/mBE0uxUp0rS7LzyiZKek7RC0lyle9sU61c5c1P95ZImNGCMRyW9LOmZtA1J5ZdI2pRX/tnWep/MzKxhslyhXQssiYixwJJ0fAhJA4HrgNOBScB1eYnvduAyYGzaZtTT7/l5deek9vWNAXBxRIxP28a88p/lld/VjPfBzMxaQJYJbSYwP+3PBy4sUGc6UB4RWyJiK1AOzJA0DOgbEUsjIoAFee2L9TsTWBA5S4H+qZ+CY7ToTM3MrNVlmdCGRsT6tP86MLRAneHA2rzjdalseNqvXV5Xv3X1Vai8xg/SacV/qDmtmXw4nbq8T9LIYpOUNEdShaSKTZs2FatmZmbN1KoJTdJiSc8X2Gbm10urrGjp8Vug34sj4l3AWWn7VCr/b2BURJxCbkU3v0h7ImJeRJRFRNngwYObEYqZmdWlVRNaREyNiJMLbPcDG9IpP9LfjQW6qAbyVz8jUll12q9dTh391tVXoXIioubvDuDH5L5jIyLeiIg9qf5dwMSGvB9mZtZ6svxh9UJgNnBj+nt/gTqLgH/Ku0hjGvC1iNgiabukycCTwCzgO/X0uxC4UtJPyV0Asi0i1ksqOIakrkD/iNgsqRvwAWAx5BJl3mnNDwJ/bMiEly1btlnSmobULWAQsLmJbdsbz6X9KZV5gOfSXjVnLsc2qFZEZLIBR5K7CrGKXKIYmMrLgLvy6n0GWJG2T+eVlwHPAyuB2wDV06+A76b6zwFldY0B9AGWAcuBF4BbgS7ptf+Xyp4FHgFObIP3qyKrz8pzKf25lMo8PJf2u7XFXGqSgLVzkioioizrOFqC59L+lMo8wHNpr9piLr5TiJmZlQQntI5jXtYBtCDPpf0plXmA59JetfpcfMrRzMxKgldoZmZWEpzQzMysJDihtQOSZqS7+q+QVOgmzUXv7l/sSQFZaOY8DuSVL2zbyP9SfXNJdS6S9KKkFyT9OK+83XwmKZ7mzKVDfS6SbsmLt1LSm3mvtZvPpZnz6GifyTGSHpH0h3S7wAvyXvtaaveypOnNDibr3yZ09g3oQu63cccD3cn9tu2dtepcAtxWoO1AYFX6OyDtD+ho80iv7cz6s2jkXMYCf6h5v4Eh7e0zae5cOuLnUqv+F4G729vn0px5dMTPhNzFIFek/XcCq/P2nwV6AMelfro0Jx6v0LI3CVgREasiYi/wU3JPBmiI9vSkgObMo71pyFwuA76b3nfiz48Wak+fCTRvLu1NY/8b+wTwk7Tfnj6X5syjvWnIXALom/b7Aa+l/ZnATyNiT0S8Qu7GFpOaE4wTWvbqu9t/jUJ3929o27bQnHkA9ExPJVgqqdCjhNpSQ+YyDhgn6bcp5hmNaNuWmjMX6HifCwCSjiX3f/0PN7ZtG2jOPKDjfSbfAD4paR3wALkVZ0PbNooTWsfQ4Lv7t3N1zePYyN1F4G+Bb0sanUWAjdCV3Km6KeT+D/pOSf0zjajp6ppLR/tcanwcuC8iDmQdSDMVmkdH+0w+AdwTESOAC4AfSmqV3OOElr2id/uvEcXv7l9v2zbUnHkQf36ywSrgUeC01gy2Hg15X9cBCyNiXzpdUkkuKbSnzwSaN5eO+LnU+DiHnqZrT59Lc+bRET+TS4F7ASLiCaAnuRsVt/xnkvWXip19I/d/x6vInVao+VL1pFp1huXt/w2wNO0PBF4h9yX3gLQ/sAPOYwDQI+0PIndj6aJfkreTucwA5ufFvJbcjbHbzWfSAnPpcJ9LqncisJp044hU1m4+l2bOo8N9JsCDwCVp/x3kvkMTcBKHXhSyimZeFJLJm+DtL/6juIDc/xWvBL6eyq4HPpj2i97dnyJPI+hI8wDeTe4JCM+mv5d2gM9EwM3Aiynmj7fHz6Q5c+mIn0s6/gZwY4G27eZzaeo8OuJnQu5qxt+mmJ8BpuW1/Xpq9zJwfnNj8a2vzMysJPg7NDMzKwlOaGZmVhKc0MzMrCQ4oZmZWUlwQjMzs5LghGZWIiR9Q9JX20EcqyUNyjoO63yc0MzMrCQ4oZm1Y5L6SPqVpGclPS/pY/krIEllkh7Na3KqpCfSM78uS3WGSXosPT/reUlnpfLb001uX5D0f/LGXC3p/6X6FZImSFokaaWky1OdKanPX6VnWd1R6P58kj4p6anU1/ckdWnN98s6Nyc0s/ZtBvBaRJwaEScD/1NP/VOAc4AzgH+UdDS5m9guiojxwKnk7tYAubs6lKU275V0Sl4/r6b6jwP3AB8BJgP/J6/OJHJ3Tn8nMBr4UH4gkt4BfAx4T+rrAHBxI+Zu1ihdsw7AzOr0HHCTpH8GfhkRj0uqq/79EfE28LakR8glnaeBuyV1A/4rImoS2kWS5pD7d2AYucS0PL1W8yTk54DDI2IHsEPSnrw78T8VuRvkIuknwJnAfXmxnEvuBtRPp5h7Ae31WWtWApzQzNqxiKiUNIHc/fK+KWkJsJ8/n13pWbvJX3YRj0k6G3g/cI+km8mtvL4K/FVEbJV0T62+ap6KcDBvv+a45t+Nvxir1rHI3fT4a/VM06xF+JSjWTuWThnuiogfAf8CTCB3B/aaR+98uFaTmZJ6SjqS3PPNnk4PidwQEXeSe2zPBHJPEH4L2CZpKHB+E8KbJOm49N3Zx4Df1Hp9CfARSUPSXAamWMxahVdoZu3bu4B/kXQQ2AdcQe7U3fcl3UDueVj5lpN7ksEg4IaIeE3SbODvJO0DdgKzIuIVSX8AXiL3uJjfNiG2p4HbgDFpzF/kvxgRL0r6e+ChlPT2AV8A1jRhLLN6+W77ZtZokqYAX42ID2Qdi1kNn3I0M7OS4BWamZmVBK/QzMysJDihmZlZSXBCMzOzkuCEZmZmJcEJzczMSsL/B9RVXyh4DoF8AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "plt.plot(subsample_s, test_means)\n",
    "plt.xlabel( 'subsample' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最优值落在边界上，暂时不再继续调，subsample=0.8，待确定其他参数后再收缩范围查找"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "6）调学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   46.8s\n",
      "[Parallel(n_jobs=5)]: Done  16 out of  20 | elapsed:  1.3min remaining:   19.8s\n",
      "[Parallel(n_jobs=5)]: Done  20 out of  20 | elapsed:  1.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.8, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "        ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=0.8, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'learning_rate': [0.001, 0.01, 0.1, 1]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'num_leaves':70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':0.8,\n",
    "          'subsample':0.8\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "learning_rate_s = [0.001,0.01,0.1,1]\n",
    "tuned_parameters = dict( learning_rate = learning_rate_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.2041890741367008\n",
      "{'learning_rate': 0.001}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAELCAYAAAAoUKpTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XeYFfX5/vH3swWWjnSUjiBSpC1IWzSxAaKAimLUiA0rJYnmqzHFxDSTqCyiIvYasIuAEDtLlaV3pApYWJAqSH1+f5xjwg8XtnDOzin367rO5Tkzc2buEdhnZ56Z+Zi7IyIicjwpQQcQEZHYp2IhIiIFUrEQEZECqViIiEiBVCxERKRAKhYiIlIgFQsRESmQioWIiBRIxUJERAqUFnSASKlWrZo3aNAg6BgiInFlzpw5W9y9ekHLJUyxaNCgAbm5uUHHEBGJK2a2vjDL6TSUiIgUSMVCREQKFNPFwsx6mNkKM1tlZncHnUdEJFnFbLEws1TgUaAn0By40syaB5tKRCQ5xWyxADoCq9x9jbvvB8YAfQLOJCKSlGK5WJwCbDji88bwNBERKWGxXCwKZGaDzCzXzHLz8vKCjiMikrBiuVhsAuoe8blOeNp/uftod89098zq1Qu8pyRfe/Yf5NGPV3Hw0OHiJxURSXCxXCxmA03MrKGZlQIGAOMivZGPlm/mn5NXcP/4pZFetYhIwojZO7jd/aCZ3QFMBlKBZ9x9SaS30/uMk1mwYTtP5qylYbVyDOzasEjfP3zY2bXvIDv3HmDH3gPs/P4AO/ceYOfeg//9vGPvgSPmH/zv530HD3Nt5/oMOacJaamxXLdFJNnFbLEAcPeJwMRob+funqezbuse/jR+KRnpqdSsmPHjH/JH//D//gA79hxg176DuB973SkGFcukUzEjnUpl0qlYJo2aFctTMSOdb7/bz4iPVjF11RayB7SlbpWy0d5VEZFiMT/eT7o4kpmZ6SfybKg9+w9y+RMzWLxp54/mZaSnhH7Q//cHfvi/GWn//fy/af8rCpXKpFOuVBopKXbM7b674Et+89Yi3OH+vi3o17ZOsfdBRKSozGyOu2cWuJyKxf98t+8gc7/YRrnSaf/9wV+xTBql01IjlDJ/G7ft4Rdj5zN73Tb6tDmZ+/u2pGJGelS3KSICKhZx59Bh57GPVzH8w8+pXSmD7AFtaF+/StCxRCTBFbZYqKsaI1JTjMHnNOG1WzpjBv1HzWD4Byt1Sa+IxAQVixjTrt5JTBySRd82pzD8g88ZMHomG77dE3QsEUlyKhYxqEJGOg9d0YbsAW1Y8fUuemXn8M78TQV/UUQkSlQsYlifNqcwcWgWp9WqwNAx8/nF2Pns+v5A0LFEJAmpWMS4ulXKMmZQJ4ad24R35m+i14gc5n6xLehYIpJkVCziQFpqCsPObcprt3TGPdT8HvHh5xw6nBhXsolI7FOxiCPt61dh4tAsLjqjNg+9v5IBo2ewcZua3yISfSoWcaZiRjrDB7Tl4Stas+yrXfTMzuHdBV8GHUtEEpyKRZzq17YOE4dkcWqN8gz+9zx+9eoCdu87GHQsEUlQKhZxrF7Vsrx2c2eGnNOEt+Zt5MIROczfsD3oWCKSgFQs4lxaagq/PK8pY2/uzMFDzqWPT2fkR2p+i0hkqVgkiA4NQs3vXq1q86//rOTKJ2fy5fa9QccSkQShYpFAKpVJZ8SANjzYvzVLNu2gx/ApTFj4VdCxRCQBxGSxMLN/mtlyM1toZm+ZWeWgM8ULM+PS9nWYODSLRtXLc/src7nrtQV8p+a3iJyAmCwWwPtAS3c/A1gJ3BNwnrhTv2o5XrulM3f85FRenxtqfi9Q81tEiikmi4W7/8fdf/hVeCag4eOKIT01hTsvOI0xN3Vi/8HDXPr4dB77ZJWa3yJSZDFZLI5yPfBefjPMbJCZ5ZpZbl5eXgnHih9nNqrKe0O7c0HLWvxj0gquemomX+1Q81tECi+wkfLM7AOgVj6z7nX3d8LL3AtkApd4AUHjfaS8kuDuvDZnI/eNW0J6agp/v6QVPVvVDjqWiASosCPlpZVEmPy4+7nHm29mA4HewDkFFQopHDPj8sy6dGhQhWFj5nHry3MZ0KEuv7+oOWVLBfZXQUTiQEyehjKzHsCvgYvdXU/Ki7CG1crx+q1duO3sxozN3UDvEVNZtHFH0LFEJIbFZLEARgIVgPfNbL6ZjQo6UKJJT03h1z2a8cqNndiz/xCXPD6NUZ+u5rCa3yKSj8B6FpGmnkXxbd+zn3veXMR7i7+m66lVebB/G2pVygg6loiUgML2LGL1yEJKUOWypXjsqnY8cGkr5q7fTo/sKUxe8nXQsUQkhqhYCBBqfl/RoR4ThnSj7kllufnFOdzz5iL27Ned3yKiYiFHaVS9PG/c2oWbz2rEmNlf0PuRqSzepOa3SLJTsZAfKZWWwj09T+flG87ku30H6ffYNJ6cskbNb5EkpmIhx9Tl1GpMGtqdnzarwV8mLuPaZz9j887vg44lIgFQsZDjOqlcKUZd3Z6/9mvF7HXfcsHwKby/9JugY4lICVOxkAKZGT87sx7jB2dxcuUy3PRCLr99exF79x8KOpqIlBAVCym0U2uU583bujCoeyNemvkFF42cytIvdwYdS0RKgIqFFEnptFR+0+t0XrrhTHbuPUDfR6fxVI6a3yKJTsVCiqVbk2pMGtad7k2r8+cJyxj43Gw271LzWyRRqVhIsVUpV4onf96eP/dtyWdrt9JzeA4fLlPzWyQRqVjICTEzru5Un/GDu1GjYgY3PJ/L799ZzPcH1PwWSSQqFhIRp9aowNu3d+GGbg15YcZ6Lh45leVfq/ktkihULCRiSqel8rvezXnh+o5s23OAi0dO49lpa0mUJxuLJDMVC4m47k2rM2loFlmnVuOP7y7luudmk7drX9CxROQExHSxMLNfmZmbWbWgs0jRVC1fmqeuzeT+Pi2YsXorPbOn8PHyzUHHEpFiitliYWZ1gfOBL4LOIsVjZlzTuQHvDu5GtfKlue652dw3boma3yJxKGaLBfAwoXG4dcI7zjWtWYG3b+/KdV0b8Nz0dfR9dBorvt4VdCwRKYKYLBZm1gfY5O4Lgs4ikZGRnsofLmrBc9d1YMvu/Vw0cirPT1+n5rdInAhsDG4z+wColc+se4HfAOe7+w4zWwdkuvuWfNYxCBgEUK9evfbr16+PYmKJlC2793HXawv4eEUe5zSrwQOXnUG18qWDjiWSlAo7BndgxeJYzKwV8CGwJzypDvAl0NHdjzkwdGZmpufm5pZAQokEd+f56ev463vLqZiRzoOXt+asptWDjiWSdApbLGLuNJS7L3L3Gu7ewN0bABuBdscrFBJ/zIyBXRsy7o6uVC1Ximuf+Yw/vbtUzW+RGBVzxUKSS7NaFXnnjq4M7NKAZ6atpe+j0/j8GzW/RWJNzBeL8BHGj/oVkjgy0lO57+IWPDMwk7xd++j9yFRenLlezW+RGBLzxUKSx0+b1eS9YVl0alSV3729mJtemMPW3brzWyQWqFhITKlRIYNnB3bg972bM2VlHj2yc8j5PC/oWCJJT8VCYk5KinF9t4a8c0dXKpdJ55qnP+PP45ey76Ca3yJBUbGQmHV67Yq8O7gb13Sqz1NT19Lv0ems2rw76FgiSUnFQmJaRnoq9/dtyVM/z+Trnd/T+5EcXp6l5rdISVOxkLhwbvOaTBqaRYcGVbj3rcUMenEO3363P+hYIklDxULiRo2KGTx/XUd+e+HpfLoij57ZU5i2SldVi5QEFQuJKykpxo1ZjXjr9i6UL53G1U/P4m8Tl7H/4OGgo4kkNBULiUstTq7E+MFZ/KxjPZ6YsoZLHp/G6jw1v0WiRcVC4laZUqn8pV8rRl/Tnk3b9tJ7xFTGfPaFmt8iUaBiIXHv/Ba1mDSsO+3qV+buNxdx60tz2b5HzW+RSFKxkIRQs2IGL15/Jr/p1YwPl39Dj+E5TF+t5rdIpKhYSMJISTEGdW/MW7d1pWzpVK56ahZ/f2+5mt8iEaBiIQmn5SmVGD+4GwM61GPUp6u59PHprFHzW+SEqFhIQipbKo2/XdKKUVe3Z8O2PfR+ZCqvzt6g5rdIMalYSELr0bIWk4Z2p03dyvz6jYXc/spcduw5EHQskbgTs8XCzAab2XIzW2Jm/wg6j8SvWpUyeOmGM7m7ZzP+s+QbemRPYeaarUHHEokrMVkszOwnQB+gtbu3AP4VcCSJcykpxi1nhZrfGempXPnkTP45eTkHDqn5LVIYhS4WZvYPM6toZulm9qGZ5ZnZ1VHKdSvwd3ffB+Dum6O0HUkyreqEmt+Xt6/Lox+v5rJRM1i35bugY4nEvKIcWZzv7juB3sA64FTgrmiEApoCWWY2y8w+NbMOUdqOJKFypdN44LIzePyqdqzb8h0XjsjhtVw1v0WOJ60Yy14IvObuO8ys2Bs2sw+AWvnMuje8rSpAJ6AD8KqZNfKj/jWb2SBgEEC9evWKnUWSU89WtWldtzK/fHU+d72+kE9W5vHXfq2oVCY96GgiMccK+9uUmf0d6AvsBToClYHx7n5mxEOZTQIecPePw59XA53c/ZiDMWdmZnpubm6ko0gSOHTYGfXpah5+fyU1K2bw8BVt6NiwStCxREqEmc1x98yCliv0aSh3vxvoAmS6+wHgO0JN6Gh4G/gJgJk1BUoBenaDREVqinH7T07ljVu7kJ5qDBg9gwf/s0LNb5EjFKXB3R844O6HzOy3wEvAyVHK9QzQyMwWA2OAa48+BSUSaa3rVmbCkCwubVeHRz5aRf9RM1i/Vc1vEShag/t37r7LzLoB5wJPA49HI5S773f3q929pbu3c/ePorEdkaOVK53GP/u3ZuTP2rI6bze9snN4c+5GNb8l6RWlWBwK//dCYLS7TyB0ekgk4fQ+42QmDetOi5Mr8ctXFzB0zHx2fq87vyV5FaVYbDKzJ4ArgIlmVrqI3xeJK6dULsO/B3XizvObMmHRV/QcnkPuum+DjiUSiKL8sL8cmAxc4O7bCV3aGq37LERiQmqKccdPm/D6LZ1JTTEuf2IGD7+/koNqfkuSKcrVUHuA1cAFZnYHUMPd/xO1ZCIxpG29k5g4NIt+beuQ/eHnXDF6Jhu+3RN0LJESU5SroYYCLwM1wq+XzGxwtIKJxJrypdN48PLWjLiyLSu/2UWv7Bzenrcp6FgiJaIoN+UtBDq7+3fhz+WAGe5+RhTzFZpuypOStHHbHn4xdj6z122jb5uT+VPfllTM0J3fEn8iflMeYPzviijC74v/vA+ROFbnpLL8+6ZO/PK8pry78Ct6ZecwZ/22oGOJRE1RisWzwCwzu8/M7gNmErrXQiQppaWmMOScJrx6c2fM4PInZpD9wedqfktCKkqD+yHgOuDb8Os6dx8erWAi8aJ9/ZOYOCSLi1ufzMMfrGSAmt+SgArsWZjZcZ+o5u4xceG5ehYSC96Zv4nfvrUYgD/3a0mfNqcEnEjk+ArbsyjMI8rnAM7/+hM/VBcLv29UrIQiCahPm1NoV+8kho2dz9Ax8/l0ZR5/6tOS8qWLMhqASOwp8G+wuzcszIrMrIW7LznxSCLxrW6Vsowd1IlHPlrFIx99Tu66bWQPaEPbeicFHU2k2CL5uI4XI7gukbiWlprCL85ryqs3d+bQYeeyUTN45MPPOXRYDySU+BTJYqHLaEWOktmgCu8Ny6L3GbV58P2VXDl6Jpu27w06lkiRRbJY6FcmkXxUzEgne0BbHr6iNUu/2knP4VMYv/DLoGOJFImeGitSQvq1rcPEIVk0rlGeO16Zx52vLWD3voNBxxIplEgWi/2RWpGZtTGzmWY238xyzaxjpNYtEqR6Vcvy6s2dGfLTU3lz7kYuHJHD/A3bg44lUqCiPEiwXT6vxmaWBuDunSKY6x/AH929DfD78GeRhJCemsIvzz+NMYM6c/CQc9nj03n041VqfktMK8qRxWOEHvExGngSmAG8Bqwws/MjnMuBiuH3lQCd4JWE07FhFSYOzaJHy1r8c/IKfvbkTL5U81tiVFGKxZdAW3fPdPf2QFtgDXAekf/NfxjwTzPbAPwLuCe/hcxsUPg0VW5eXl6EI4hEX6Uy6TxyZVv+1b81izftoGd2DhMXfRV0LJEfKcojyhe7e8v8ppnZ/PApo8Jv2OwDoFY+s+4FzgE+dfc3zOxyYJC7n3u89elxHxLv1m35jqFj57Ngw3Yuz6zDHy5qQTnd+S1RVtjHfRSlWIwl9ADBMeFJVwDVgGuAqe7eoZhZ89vWDqCyu7uZGbDD3Sse7zsqFpIIDhw6TPYHn/PoJ6toULUc2QPacEadykHHkgQWjfEsBgKrCJ0iGkboFNRA4ADwk6JHPK4vgbPC738KfB7h9YvEpPTUFO684DT+fVMnvj9wiEsem87jn6zmsJrfErBCH1kAmFkp4DRCDegV7n4gKqHMugHZhJ5d9T1wm7vPOd53dGQhiWbHngP85q1FTFj0FZ0bVeWhK1pTu1KZoGNJgonGaaizgeeBdYQe7VEXuNbdpxQ/ZuSoWEgicndem7OR+8YtIT01hQcubUWPlrWDjiUJJBqnoR4Eznf3s9y9O3AB8HBxA4pIwcyMyzPrMmFIFvWrluWWl+Zyz5sL2bNfd35LySpKsUh39xU/fHD3lYBGqBcpAQ2rleP1W7pw69mNGTN7A70fmcriTTuCjiVJpCjFItfMnjKzs8OvJwGd9xEpIaXSUvi/Hs14+cYz2bPvEP0em8YTn6r5LSWjKMXiVmApMCT8WhqeJiIlqEvjakwalsU5zWryt/eWc80zs/hm5/dBx5IEV6SroWKZGtySbNydsbM38Md3l5KRnsIDl57B+S3yu89V5NgiNga3mS3iOGNVuPsZRcwmIhFgZgzoWI8ODaswdMw8Br04h5+dWY/fXdicMqVSg44nCaYwzxLoHfUUIlJsjauX581bu/Lg+ysYPWUNs9ZsJXtAW1qeUinoaJJACuxZuPv6471+WM7MZkQ3qogcS6m0FO7peTov3XAmu/cdpN9j03gqZ42a3xIxkRz8KCOC6xKRYuh6ajUmDe3OT06rwZ8nLOPaZz9js5rfEgEag1skwZxUrhRPXNOev/Zrxex139IjO4cPln4TdCyJcxqDWyQBmRk/O7Me4wdnUatiBje+kMvv3l7M9wcOBR1N4lQki4VFcF0iEgGn1ijPW7d34aashrw4cz0XPTKVZV/tDDqWxKFIFotrIrguEYmQ0mmp3Hthc168oSM79h6gz8hpPD11rZrfUiSFLhZmtsvMdh712mBmb5lZI3dfHM2gInJisppUZ9Kw7nRvWp37xy9l4HOz2bxLzW8pnKIcWQwH7gJOAeoAdwKvEBo575nIRxORSKtSrhRP/rw9f+7bkllrttJzeA4fLVfzWwpWlGJxsbs/4e673H2nu48GLnD3scBJRd2wmfU3syVmdtjMMo+ad4+ZrTKzFWZ2QVHXLSLHZmZc3ak+4wd3o0bFDK5/Lpc/vKPmtxxfUYrFHjO73MxSwq/LCY1iB8W7bHYxcAnw/w2eZGbNgQFAC6AH8JiZ6dkFIhHWpGYF3r69Czd0a8jzM9bTZ+Q0ln+t5rfkryjF4ipCTezNwDfh91ebWRngjqJu2N2XHTk+xhH6AGPcfZ+7ryU07nfHoq5fRApWOi2V3/VuzvPXd2Trd/u5eOQ0np22lkR5wKhETqGLhbuvcfeL3L2au1cPv1/l7nvdfWoEM50CbDji88bwNBGJkrOaVmfSsCy6nVqNP767lOufm82W3fuCjiUxpChXQzU1sw/NbHH48xlm9tsCvvOBmS3O59XnRIOH1z/IzHLNLDcvLy8SqxRJWtXKl+bpazP5U58WTF+9lR7Dp/Dxis1Bx5IYUZTTUE8C9wAHANx9IaHewjG5+7nu3jKf1zvH+domoO4Rn+uEp+W3/tHununumdWrVy/CrohIfsyMn3duwLuDu1GtfGmue3Y2941boua3FKlYlHX3z46aFo1R48cBA8ystJk1BJoAR29XRKKoac0KvH17VwZ2acBz09fR99FprPxmV9CxJEBFKRZbzKwx4SufzOwy4KvibtjM+pnZRqAzMMHMJgO4+xLgVULDtk4Cbnd3/VojUsIy0lO57+IWPHtdB7bs3sdFj0zlhRnr1PxOUoUeVtXMGgGjgS7ANmAtcNWRY1oEScOqikRP3q593PX6Aj5Zkcc5zWrwj8vOoGr50kHHkggo7LCqRTmy2AQ8C/yF0F3b7wPXFi+eiMST6hVK8+zADtx3UXNyVm3hguE5fLpSF5Ukk6IUi3eAiwg1uL8EdgPfRSOUiMQeM2Ng14aMu6MrVcqlc+0zn3H/+KXsO6izxMmgMGNw/6COu/eIWhIRiQvNalVk3B3d+NvEZTw9dS3TV29lxIA2NKlZIehoEkVFObKYbmatopZEROJGRnoqf+zTkmcGZrJ55/f0fmQqL85cr+Z3AitKsegGzAk/3G+hmS0ys4XRCiYise+nzWry3rAsOjWqyu/eXsxNL8zh2+/2Bx1LoqAoV0PVz2+6roYSkcOHnWenr+OB95ZTuWw6D13ehm5NqgUdSwoh4ldDufv6/F4nFlNEEkFKinFDt4a8fXtXKpVJ5+qnZ/GXCWp+J5JIDqsqIkmu+cmh5vc1nerzZM5aLnlsOqs27w46lkSAioWIRFSZUqnc37clT/48ky+376X3Izm8MusLNb/jnIqFiETFec1rMnlYdzo0qMJv3lrELS/NYZua33FLxUJEoqZGxQyev64jv73wdD5avpke2VOYtmpL0LGkGFQsRCSqUlKMG7Ma8dZtXSlfOo2rn57F3yYuY//Bw0FHkyJQsRCREtHylEqMH5zFlR3r8cSUNVz6+HTW5Kn5HS9ULESkxJQplcpf+7XiiWvas2HbHi4cMZUxn6n5HQ9ULESkxF3QohaTh3WnXf3K3P3mIm57eS7b96j5HctULEQkEDUrZvDi9WdyT89mfLDsG3oMz2HG6q1Bx5JjCKxYmFl/M1tiZofNLPOI6eeZ2Zzws6fmmNlPg8ooItGVkmLcfFZj3rqtK2VLpfKzp2bywKTlHDik5nesCfLIYjFwCTDlqOlbgIvcvRWhwZVeLOlgIlKyWp5SifFDujGgQ10e/2Q1lz4+nbVbNFxOLAmsWLj7Mndfkc/0ee7+ZfjjEqCMmWn8RpEEV7ZUGn+75AxGXd2O9Vv3cOGIHF6dvUHN7xgR6z2LS4G57r4v6CAiUjJ6tKzNpGFZtK5TmV+/sZA7XpnHjj0Hgo6V9KJaLMzsAzNbnM+rTyG+2wJ4ALj5OMsMMrNcM8vNy9N4wCKJonalMrx045n8X49mTF7yNT2zpzBrjZrfQSr0eBZRC2D2CXCnu+ceMa0O8BFwnbtPK8x6NJ6FSGJasGE7w8bOZ93W77jt7MYMO7cp6amxflIkfkR8PIuSYmaVgQnA3YUtFCKSuFrXrcz4wd3o374Oj368mstGzWD9VjW/S1qQl872M7ONQGdggplNDs+6AzgV+L2ZzQ+/agSVU0SCV650Gv+4rDWPXdWOtXm76ZWdw+tzNqr5XYICPw0VKToNJZIcvty+l1+Mnc+std/S+4za/KVfKyqVSQ86VtyK29NQIiLHc3LlMrxyUyfuuuA0Ji3+ml7ZOXy29tugYyU8FQsRiTupKcbtPzmV12/tQlqqMWD0DB76zwoO6s7vqFGxEJG41aZuZSYMyeKSdnUY8dEq+j8xgy+27gk6VkJSsRCRuFa+dBr/6t+aR65sy6rNu+k1Ioe35m0MOlbCUbEQkYRwUeuTeW9oFs1rV+QXYxcwdMw8dn6vO78jRcVCRBJGnZPK8u9BnfjVeU0Zv/AremXnMGe9mt+RoGIhIgklNcUYfE4TXrulM2bQf9QMHn5/pZrfJ0jFQkQSUrt6JzFxSBZ925xC9oefc8XomWz4Vs3v4lKxEJGEVSEjnYeuaEP2gDas/HoXvbJzeGf+pqBjxSUVCxFJeH3anMLEoVmcVqsCQ8fM5xdj57NLze8iUbEQkaRQt0pZxgzqxLBzm/DO/E30GpHDnPXbgo4VN1QsRCRppKWmMOzcprx2S2fc4fInZjDiw885dDgxnpEXTSoWIpJ02tevwsShWVx0Rm0een8lA0bPYOM2Nb+PR8VCRJJSxYx0hg9oy/Ar2rDsq130zM5h3IIvg44Vs1QsRCSp9W17ChOHZNGkRnmG/Hsev3p1Abv3HQw6VswJcvCj/ma2xMwOm9mPnqVuZvXMbLeZ3RlEPhFJHvWqluXVmzsz5JwmvDVvIxeOyGHeF2p+HynII4vFwCXAlGPMfwh4r+TiiEgyS0tN4ZfnNWXszZ05eMi5bNQMRn6k5vcPAisW7r7M3VfkN8/M+gJrgSUlm0pEkl2HBqHmd69WtfnXf1Zy5ZMz+XL73qBjBS7mehZmVh74P+CPQWcRkeRUqUw6Iwa04cH+rVmyaQc9hk9hwsKvgo4VqKgWCzP7wMwW5/Pqc5yv3Qc87O67C7H+QWaWa2a5eXl5EcstImJmXNq+DhOHZtGoenluf2Uud722gO+StPlt7sGejzOzT4A73T03/DkHqBueXRk4DPze3Ucebz2ZmZmem5sbzagikqQOHDrMiA8/Z+THq6hfpSzZA9rSum7loGNFhJnNcfcfXWR0tJg7DeXuWe7ewN0bAMOBvxZUKEREoik9NYVfnX8aY27qxP6Dh7n08ek89smqpGp+B3npbD8z2wh0BiaY2eSgsoiIFMaZjary3tDuXNCyFv+YtIKrnprJVzuSo/kd+GmoSNFpKBEpKe7O63M28odxS0hPTeHvl7SiZ6vaQccqlrg9DSUiEuvMjP6ZdZk4JIsGVcty68tzufuNhezZn7jNbxULEZFialCtHK/f2oXbzm7M2NwN9B4xlUUbdwQdKypULERETkB6agq/7tGMV27sxN4Dh7jk8WmM+nQ1hxOs+a1iISISAZ0bV+W9oVmc17wmf39vOVc/PYuvd3wfdKyIUbEQEYmQymVL8ejP2vHApa2Y98V2emRPYfKSr4OOFREqFiIiEWRmXNGhHhOGdKPuSWW5+cU53PPmorhvfqtYiIhEQaPq5Xnj1i7cclZjxsz+gt66qqX0AAAJ10lEQVSPTGXxpvhtfqtYiIhESam0FO7u2YyXbzyTPfsO0e+xaTw5ZU1cNr9VLEREoqxL42q8NzSLnzarwV8mLuPaZz/jm53x1fxWsRARKQEnlSvFqKvb87dLWpG7bhs9hk/h/aXfBB2r0FQsRERKiJlxZcd6jB/SjZMrl+GmF3K5961F7N1/KOhoBVKxEBEpYY2rl+fN27owqHsjXp71BReNnMrSL3cGHeu4VCxERAJQOi2V3/Q6nZduOJOdew/Q99FpPJUTu81vFQsRkQB1a1KNScO6c9Zp1fnzhFDze3MMNr9VLEREAlalXClGX9Oev/Rryex139IjO4cPl8VW81vFQkQkBpgZV51Zn/GDu1GzYgY3PJ/L799ZzPcHYqP5HeRIef3NbImZHTazzKPmnWFmM8LzF5lZRlA5RURK0qk1KvD27V24sVtDXpixnotHTmXZV8E3v4M8slgMXAJMOXKimaUBLwG3uHsL4GzgQImnExEJSOm0VH7buzkvXN+RbXsO0OfRaTwzdS1BjmwaWLFw92XuviKfWecDC919QXi5re4eG8dhIiIlqHvT6kwamkX3JtX40/ilDHx2Nnm79gWSJRZ7Fk0BN7PJZjbXzH4ddCARkaBULV+aJ3+eyf19WjBzzVZ6Zk/h4+WbSzxHVIuFmX1gZovzefU5ztfSgG7AVeH/9jOzc46x/kFmlmtmuXl5eVHYAxGR4JkZ13RuwLuDu1GtfGmue242941bUqLN77Rortzdzy3G1zYCU9x9C4CZTQTaAR/ms/7RwGiAzMzM2LyTRUQkQprWrMDbt3flH5NW8My0tcxYvZURV7bltFoVor7tWDwNNRloZWZlw83us4ClAWcSEYkJGemp/P6i5jx3XQe2frefi0ZO5ZVZX0R9u0FeOtvPzDYCnYEJZjYZwN23AQ8Bs4H5wFx3nxBUThGRWHT2aTWYNCyLro2rkpZiUd+eBXkpViRlZmZ6bm5u0DFERErUDz/DzYpXMMxsjrtnFrRcVHsWIiISXcUtEkUViz0LERGJMSoWIiJSIBULEREpkIqFiIgUSMVCREQKpGIhIiIFUrEQEZECJcxNeWaWB6wv5terAVsiGCceaJ+Tg/Y5OZzIPtd39+oFLZQwxeJEmFluYe5gTCTa5+SgfU4OJbHPOg0lIiIFUrEQEZECqViEjA46QAC0z8lB+5wcor7P6lmIiEiBdGQhIiIFSqpiYWY9zGyFma0ys7vzmV/azMaG588yswYlnzKyCrHPvzSzpWa20Mw+NLP6QeSMpIL2+YjlLjUzN7O4v3KmMPtsZpeH/6yXmNkrJZ0x0grxd7uemX1sZvPCf797BZEzUszsGTPbbGaLjzHfzGxE+P/HQjNrF9EA7p4ULyAVWA00AkoBC4DmRy1zGzAq/H4AMDbo3CWwzz8Byobf35oM+xxergIwBZgJZAaduwT+nJsA84CTwp9rBJ27BPZ5NHBr+H1zYF3QuU9wn7sD7YDFx5jfC3gPMKATMCuS20+mI4uOwCp3X+Pu+4ExQJ+jlukDPB9+/zpwjpXUyCLRUeA+u/vH7r4n/HEmUKeEM0ZaYf6cAe4HHgC+L8lwUVKYfb4JeNRDwxbj7ptLOGOkFWafHagYfl8J+LIE80Wcu08Bvj3OIn2AFzxkJlDZzGpHavvJVCxOATYc8XljeFq+y7j7QWAHULVE0kVHYfb5SDcQ+s0knhW4z+HD87qeOGO7F+bPuSnQ1MymmdlMM+tRYumiozD7fB9wtZltBCYCg0smWmCK+u+9SDSsqgBgZlcDmcBZQWeJJjNLAR4CBgYcpaSlEToVdTaho8cpZtbK3bcHmiq6rgSec/cHzawz8KKZtXT3w0EHi0fJdGSxCah7xOc64Wn5LmNmaYQOXbeWSLroKMw+Y2bnAvcCF7v7vhLKFi0F7XMFoCXwiZmtI3Rud1ycN7kL8+e8ERjn7gfcfS2wklDxiFeF2ecbgFcB3H0GkEHoGUqJqlD/3osrmYrFbKCJmTU0s1KEGtjjjlpmHHBt+P1lwEce7hzFqQL32czaAk8QKhTxfh4bCthnd9/h7tXcvYG7NyDUp7nY3XODiRsRhfm7/TahowrMrBqh01JrSjJkhBVmn78AzgEws9MJFYu8Ek1ZssYBPw9fFdUJ2OHuX0Vq5UlzGsrdD5rZHcBkQldSPOPuS8zsT0Cuu48DniZ0qLqKUCNpQHCJT1wh9/mfQHngtXAv/wt3vziw0CeokPucUAq5z5OB881sKXAIuMvd4/aouZD7/CvgSTP7BaFm98B4/uXPzP5NqOBXC/dh/gCkA7j7KEJ9mV7AKmAPcF1Etx/H/+9ERKSEJNNpKBERKSYVCxERKZCKhYiIFEjFQkRECqRiISIiBVKxEBGRAqlYSMIzs90lsI2Lj/c49Chvu6+ZNQ9i25I8dJ+FJDwz2+3u5SOwnlR3PxSJTJHctpk9B4x399dLNpUkEx1ZSFIxs7vMbHZ4cJg/HjH9bTObEx4YaNAR03eb2YNmtgDobGbrzOyPZjbXzBaZWbPwcgPNbGT4/XPhQWimm9kaM7ssPD3FzB4zs+Vm9r6ZTfxh3jGyrjOzB8xsLtDfzG4KZ19gZm+YWVkz6wJcDPzTzOabWePwa1J4f3J+yChyIlQsJGmY2fmEHp7XEWgDtDez7uHZ17t7e0JP3h1iZj88mr4coUFkWrv71PC0Le7eDngcuPMYm6sNdAN6A38PT7sEaEBoIJ5rgM6FiL3V3du5+xjgTXfv4O6tgWXADe4+ndAzge5y9zbuvprQoD+Dw/tzJ/BYIbYjclxJ82woEeD88Gte+HN5QsVjCqEC0S88vW54+lZCz1F646j1vBn+7xxCBSA/b4cfhb3UzGqGp3UDXgtP/9rMPi5E5rFHvG9pZn8GKoezTz56YTMrD3Thf8/6AihdiO2IHJeKhSQTA/7m7k/8fxPNzgbOBTq7+x4z+4TQE0oBvs+nV/DDY9wPcex/Q0c+6v1ERlv87oj3zwF93X2BmQ0k/BTZo6QA2929zQlsU+RHdBpKkslk4Prwb9+Y2SlmVoPQuCXbwoWiGaExLqJhGnBpuHdRk/x/2B9PBeArM0sHrjpi+q7wPNx9J7DWzPoDhB9X3fqEk0vSU7GQpOHu/wFeAWaY2SJC46xXACYBaWa2jFB/YWaUIrxBaBCipcBLwFxCQ/cW1u+AWYSKzvIjpo8B7jKzeWbWmFAhuSHclF9C/mOQixSJLp0VKUFmVt7dd4cb6J8BXd3966BziRREPQuRkjXezCoDpYD7VSgkXujIQiRgZvYW0PCoyf/n7j+62kkkKCoWIiJSIDW4RUSkQCoWIiJSIBULEREpkIqFiIgUSMVCREQK9P8ACyrQu5N4kzkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "plt.plot(learning_rate_s, test_means)\n",
    "plt.xlabel( 'learning_rate' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最优值落在边界上，暂时不再继续调，subsample=0.001，待确定其他参数后再收缩范围查找"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 9 candidates, totalling 45 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   43.8s\n",
      "[Parallel(n_jobs=5)]: Done  45 out of  45 | elapsed:  3.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "        ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'colsample_bytree': [0.8, 0.9, 1.0], 'subsample': [0.8, 0.9, 1.0]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'num_leaves':70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "colsample_bytree_s = [0.8,0.9,1.0]\n",
    "subsample_s = [0.8,0.9,1.0]\n",
    "tuned_parameters = dict( colsample_bytree = colsample_bytree_s,subsample=subsample_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.31840252504860544\n",
      "{'colsample_bytree': 1.0, 'subsample': 1.0}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 5 candidates, totalling 25 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   46.4s\n",
      "[Parallel(n_jobs=5)]: Done  22 out of  25 | elapsed:  1.9min remaining:   15.9s\n",
      "[Parallel(n_jobs=5)]: Done  25 out of  25 | elapsed:  2.0min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.1, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "        ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'learning_rate': [0.0001, 0.0004, 0.0007, 0.001, 0.0013]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'num_leaves':70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':1.0,\n",
    "          'subsample':1.0\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "learning_rate_s = [0.0001,0.0004,0.0007,0.001,0.0013]\n",
    "tuned_parameters = dict( learning_rate = learning_rate_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5,  param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.07761398042892548\n",
      "{'learning_rate': 0.0001}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "上边从num_leaves调优开始，损失变差了，现在确定完其他参数，返回继续调"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:   46.6s\n",
      "[Parallel(n_jobs=5)]: Done  16 out of  20 | elapsed:  1.5min remaining:   23.0s\n",
      "[Parallel(n_jobs=5)]: Done  20 out of  20 | elapsed:  1.5min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.0001, max_depth=3, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "     ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'num_leaves': range(50, 90, 10)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'max_depth': 3,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':1.0,\n",
    "          'subsample':1.0,\n",
    "          'learning_rate':0.0001\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "num_leaves_s = range(50,90,10) #50,60,70,80\n",
    "\n",
    "tuned_parameters = dict( num_leaves = num_leaves_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.07761398042892548\n",
      "{'num_leaves': 50}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "没有改善，再调数深度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done   8 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=5)]: Done  16 out of  20 | elapsed:  2.7min remaining:   40.4s\n",
      "[Parallel(n_jobs=5)]: Done  20 out of  20 | elapsed:  2.8min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.0001, max_depth=-1, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "    ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'max_depth': range(3, 10, 2)}, pre_dispatch='2*n_jobs',\n",
       "       refit=False, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=5)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'num_leaves': 70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':1.0,\n",
    "          'subsample':1.0,\n",
    "          'learning_rate':0.0001\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "max_depth_s = range(3,10,2) #3,5,7,9\n",
    "\n",
    "tuned_parameters = dict( max_depth = max_depth_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.07357368106073804\n",
      "{'max_depth': 9}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAELCAYAAABwLzlKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XecFdX9//HXh15EXKSXFVAQEaVdOyHGArYAUYNgCfgFEURikm+MGM3XRFJIfokxxUY0EQOChCCLJSqi2Au7sAgCgjRhWerS65bP7487G0e8sIW9zJb38/G4jztz5syZzwC7H2bmzDnm7oiIiEShWtQBiIhI1aUkJCIikVESEhGRyCgJiYhIZJSEREQkMkpCIiISGSUhERGJjJKQiIhERklIREQiUyPqAMq7xo0be9u2baMOQ0SkQsnIyNjq7k2KqqckVIS2bduSnp4edRgiIhWKma0tTj3djhMRkchEloTMrJGZzTazFcF3yhHqDQnqrDCzIUFZAzPLDH22mtnDwbaRZrYoKH/XzDoH5W3NbH9on8eP39mKiEgiUd6OGwvMcffxZjY2WL8nXMHMGgEPADHAgQwzm+Xu24FuoXoZwIxg9Vl3fzwo7wc8BFwRbFvp7v/dT0REohXl7bj+wMRgeSIwIEGdvsBsd88JEs9svkwoAJhZR6Ap8A6Au+8Kba5PPHmJiEg5FOWVUDN3zw6WNwLNEtRpBawLra8PysIGAc95aGIkMxsN/AioBVwSqtvOzBYAu4D73f2dYzsFERE5FklNQmb2OtA8wab7wivu7mZW2iuWQcAth7X3CPCImd0I3A8MAbKBVHffZmY9gZlmduZhV06FcY8ARgCkpqaWMiwRESlKUpOQu192pG1mtsnMWrh7tpm1ADYnqJYFXBxabw3MDbXRFajh7hlHOMxU4LEgloPAwWA5w8xWAh2Br/W/dvcJwASAWCym23kiIkkS5TOhWcSvUAi+0xLUeRXoY2YpQe+5PkFZocHAlPAOZtYhtHo1sCIob2Jm1YPl9kAHYFUZnIeISKVyKK+A1z7dyJvLEl0blK0onwmNB6aZ2TBgLTAQwMxiwEh3H+7uOWY2DpgX7POgu+eE2hgIXHVYu3ea2WVALrCdLxNdb+BBM8sFCoJj5CAiIhQUOB+vySEtcwMvL8pm5/5cvtGhMd/q1DSpx7XQ83xJIBaLuUZMEJHKyN1Zmr2btIVZvJC5gQ07D1CvVnX6dG5G/+6t6HVaY2pWL90NMzPLcPdYUfU0bI+ISBWzLmcfsxZuIC0zi+Wb9lCjmtG7YxPuubITl3duRr1axy81KAmJiFQBOXsP8dKibNIWZJG+djsAsVNSGDegC1ef1YJG9WtFEpeSkIhIJbXvUB6zl2wiLXMDby/fQl6B07HZCdzd93T6dW1Jm0b1og5RSUhEpDLJzS/g3c+3krYgi9eWbGLfoXxaNKzDsG+0o3/XVpzRogFmFnWY/6UkJCJSwbk787/YQVpmFi99ks22vYdoWLcm/bu1YkC3lpzTthHVqpWfxBOmJCQiUkF9vnk3MxdsIG1hFuty9lO7RjUu69yM/l1b8s3Tm1C7RvWoQyySkpCISAWycecBZi3MYuaCDSzJ3kU1g4tOa8xdl3ak75nNaFCnZtQhloiSkIhIObdzfy7/WZTNzMwsPlqdgzt0bXMS/3dNZ67p2oKmDepEHWKpKQmJiJRDB3LzeWPZZtIys3hz2RYO5RfQrnF97rq0A/27taJd4/pRh1gmlIRERMqJ/ALng5XbSMvM4pXFG9l9MI8mDWpz8/mnMKB7S85q1bBc9WwrC0pCIiIRcncWZe0kLXMDLyzcwObdBzmhdg2u6NKcAd1accGpJ1O9nPZsKwtKQiIiEVizdS9pmfGhc1Zt3Uut6tW4+PQmDOjeiks6NaVOzfLfs60sKAmJiBwnW3Yf5MVPNjAzcwML1+3ADM5r14gRvdtzZZcWNKxXsXq2lQUlIRGRJNp9IJfXPt3EzMws3vt8KwUOnVucyE+v6sS3u7akRcO6UYcYKSUhEZEydiivgLeWb2FmZhavL9nEwbwC2jSqyx0Xn0b/bi3p0KxB1CGWG5ElITNrBDwHtAXWAAPdfXuCekOA+4PVX7r7RDNrALwTqtYamOTuPwjtdx0wHTjH3dODsnuBYUA+8H13D8/SKiJSagUFzrw1OcwMTQrXqH4tbjinDf27taRHakql69lWFqK8EhoLzHH38WY2Nli/J1whSFQPADHAgQwzmxUkq26hehnAjNB6A+Au4KNQWWdgEHAm0BJ43cw6unt+ks5PRKqApdm7mJlZ9pPCVRVRJqH+wMXB8kRgLoclIaAvMLtwGm4zmw1cAUwprGBmHYGmfPXKaBzwW+Duw4431d0PAqvN7HPgXOCDsjkdEakq1m/fR1rmBmZlbuCzTbsjnRSuoovyT6qZu2cHyxuBZgnqtALWhdbXB2Vhg4DnPJin3Mx6AG3c/SUzCyehVsCHRbQlIpLQESeF638mV5/dMrJJ4Sq6pCYhM3sdaJ5g033hFXd3M/NSHmYQcEtwvGrAQ8DQUrZF0M4IYARAamrqsTQlIhVY4aRwszI38FYwKVyHpuVrUriKLqlJyN0vO9I2M9tkZi3cPdvMWgCbE1TL4stbdhDvgDA31EZXoIa7ZwRFDYAuwNzgAWBzYJaZ9QvaanNYW1lHiHsCMAEgFouVNjmKSAWUl1/AO59vZVbmBl79dOOXk8L1akf/buVvUriKLsrbcbOAIcD44DstQZ1XgV+bWUqw3ge4N7R9MKHnQ+6+E2hcuG5mc4Efu3u6me0HnjWzh4h3TOgAfFxmZyMiFVbhpHCzMrN48bBJ4fp3a8m55XhSuIouyiQ0HphmZsOAtcBAADOLASPdfbi755jZOGBesM+DhZ0UAgOBq4pzMHf/1MymAUuAPGC0esaJVG2fb95DWmYWaZkb+CJnX3xSuDOa0b9bxZkUrqKz4Hm+HEEsFvP09PSowxCRMrJx5wFeWLiBmZlZfLrhy0nh+ndrVSEnhSuvzCzD3WNF1VM/QhGp9Hbuz+WVxdnMXLCBD1dvi08K17phpZgUrqJTEhKRSulAbj5vLtvMzEo+KVxFpyQkIpVGfoHz4aptzFxQdSaFq+iUhESkQnN3FmcFQ+dUwUnhKjolIRGpkNZu28vMBRtIW5jFqi1Vd1K4ik5JSEQqjMJJ4dIyN5AZnhTuG1V3UriKTklIRMq1PQfzeHXxRtIWbuC9z7eSX+CaFK4SURISkXLnUF4BbxdOCrd0EwdyC2idUpdR3zxVk8JVMkpCIlIuFBQ46Wu3MzMzi5cXZbNjX3xSuO/2bMOA7poUrrJSEhKRSO3cn8sTb60kLXMDWTv2U7dmdfqe2Yz+3VrRq4MmhavslIREJDKfbtjJHZPns377fnp3aMxPrjhdk8JVMfqbFpFITEtfx89mLialXi2m3X4+PU9pFHVIEgElIRE5rg7k5vNA2qc8l76Oi047mT8N6k7jE2pHHZZERElIRI6bL7btY+SkDJZk72LMJafxg8s6ajSDKk5JSESOi9lLNvGjaZlUM+PvQ2Nc0qlZ1CFJOaAkJCJJlZdfwO9fW87jb63krFYNefSmHrRpVC/qsKSciKzvo5k1MrPZZrYi+E45Qr0hQZ0VZjYkKGtgZpmhz1Yze/iw/a4zMw9masXM2prZ/tA+jyf/LEWqts27D3DzUx/x+FsrufG8VP418gIlIPmKKK+ExgJz3H28mY0N1u8JVzCzRsADQAxwIMPMZrn7dqBbqF4GMCO03gC4C/josGOudPduiEjSfbw6hzufnc+uA7k8NLAr1/ZoHXVIUg5F+RZYf2BisDwRGJCgTl9gtrvnBIlnNnBFuIKZdQSaAu+EiscBvwUOlHXQInJ07s7f3l7F4L99SP3aNZg5+iIlIDmiKJNQM3fPDpY3AomeUrYC1oXW1wdlYYOA59zdAcysB9DG3V9K0F47M1tgZm+Z2TeOLXwROdyuA7mMmjSfX728lD6dmzHrzovo1PzEqMOSciypt+PM7HWgeYJN94VX3N3NzEt5mEHALcHxqgEPAUMT1MsGUt19m5n1BGaa2ZnuvitB3COAEQCpqamlDEukalmavYtRkzJYv30/9199BsN6tdNYb1KkpCYhd7/sSNvMbJOZtXD3bDNrAWxOUC0LuDi03hqYG2qjK1DD3TOCogZAF2Bu8I+/OTDLzPq5ezpwMIgrw8xWAh2B9ARxTwAmAMRisdImR5EqY3rGeu6fuYiGdWsydcT5xNpq9AMpnihvx80ChgTLQ4C0BHVeBfqYWUrQe65PUFZoMDClcMXdd7p7Y3dv6+5tgQ+Bfu6ebmZNzKw6gJm1BzoAq8r6pESqkgO5+dw74xN+/K+F9EhN4cUx31ACkhKJsnfceGCamQ0D1gIDAYIu1SPdfbi755jZOGBesM+D7p4TamMgcFUxj9cbeNDMcoGC4Bg5RewjIkewLmcfoyZnsDhrF6O/dSo/uvx0jX4gJWbB83w5glgs5unpX7tjJ1KlzVm6iR8+lwnAH2/oxqVnaPQD+Sozy3D3WFH1NGKCiBRbfoHz0OzPeOTNlXRpdSKP3dRTL5/KMVESEpFi2bL7IHdNXcD7K7cx+Nw2PPDtM6lTs3rUYUkFpyQkIkWatyY++sGOfbn8/rtdub6nXj6VsqEkJCJH5O489e5qfvOfZbRJqcvTo8/ljBZ6+VTKjpKQiCS0+0AuP5n+Cf9ZvJG+Zzbj/323KyfWqRl1WFLJKAmJyNcs27iLUZPm80XOPu676gyGf0OjH0hyKAmJyFfMmL+enz6/iBPr1GTKbedzbju9fCrJoyQkIkB89IMHX1zCsx99wfntG/Hnwd1p2qBO1GFJJackJCKsy9nHHZPnsyhrJyO/eSo/7tORGtWjHNVLqgolIZEq7s1lm/nBc5kUuPO378W4vLNGP5DjR0lIpIrKL3D+OHs5f33zczq3OJHHbu7BKSfXjzosqWKUhESqoG17DvL9qQt47/Nt3BBrwy/6a/QDiYaSkEgVk7E2h9GTF7B93yF+d/3ZDIy1iTokqcKUhESqCHfnH++t4dcvL6VVSl1m3HEhZ7ZsGHVYUsUpCYlUAXsO5nHPvz/hpU+yubxzM37/3a40rKvRDyR6kfXBNLNGZjbbzFYE3ylHqDckqLPCzIYEZQ3MLDP02WpmDwfbhprZltC24UdrS6SyW75pN/3++i6vLN7I2Cs7MeGWnkpAUm5EeSU0Fpjj7uPNbGywfk+4gpk1Ah4AYoADGWY2y923A91C9TKAGaFdn3P3O0vQlkilNHNBFvfOWMQJdWowefh5nN/+5KhDEvmKKN9G6w9MDJYnAgMS1OkLzHb3nCBZzAauCFcws45AU+CdIo5XZFsilcXBvHzun7mIHzyXyVmtG/LSmF5KQFIuRXkl1Mzds4PljUCiN+RaAetC6+uDsrBBxK98wvOUX2dmvYHlwA/dfV0x2xKp8NZv38foyfNZuH4nt/duz919T9foB1JuFftfppn9zsxONLOaZjYneO5ycxH7vG5mixN8+ofrBQnEj9BMUQYBU0LrLwBt3f1s4lc7ExPudfS4R5hZupmlb9mypZRhiRx/b362mWv+8i6rtuzliVt6cu9VZygBSblWkn+dfdx9F3ANsAY4Dbj7aDu4+2Xu3iXBJw3YZGYtAILvzQmayALCLzG0DsoI9usK1HD3jNAxt7n7wWD1SaBncdo6LO4J7h5z91iTJk2Odooi5UJ+gfPQa5/xP0/Po0XDurwwphd9z2wedVgiRSpJEiq8dXc18C9333mMx54FFPZQGwKkJajzKtDHzFKC3nN9grJCg/nqVVBhQivUD1hazLZEKqRtew4y9B8f8+c3Puf6Hq15/o4LadtYw+9IxVCSZ0IvmtkyYD8wysyaAAeO4djjgWlmNgxYCwwEMLMYMNLdh7t7jpmNA+YF+zzo7jmhNgYCVx3W7vfNrB+QB+QAQwGK0ZZIhTP/i+2MnjyfbXsP8dvrzuKGc1KjDkmkROyrz/OLqBzv5rzT3fPNrB5wortvTFp05UAsFvP09PSowxD5Cndn4vtr+NXLS2nRsC6P3tSDLq00+oGUH2aW4e6xouqVpGPCd4HcIAHdD0wCWh5DjCJSCnsP5jFmygJ+/sISvtmxKS/c2UsJSCqsktyO+5m7/8vMegGXAf8PeAw4LymRicjXrNi0m5GTMli9dS/3XNGJ23u3p1o1izoskVIrSceE/OD7amCCu78E1Cr7kEQkkbTMLPo/8h479+cyafh5jLr4VCUgqfBKciWUZWZPAJcDvzWz2kQ74oJIlXAwL59fvbSUZz5YyzltU/jrjT1odmKdqMMSKRMlSUIDiQ9z83t33xF0hT7qe0IicmyyduznjsnzWbhuB7d9ox0/uaITNfXyqVQixU5C7r7PzFYCfc2sL/COu7+WvNBEqra3lm/hB1MXkJvvPH5zD67o0qLonUQqmJL0jrsLmEx8sNCmwCQzG5OswESqqoIC5+HXlzP0Hx/T7MQ6vDCmlxKQVFoluR03DDjP3fcCmNlvgQ+AvyQjMJGqKGfvIX7wXCZvL9/CtT1a8asBZ1G3VvWowxJJmpIkIePLHnIEy+qaI1JGMtft4I5JGWzde4jfXHsWg85pg5l+xKRyK0kS+gfwkZk9H6wPAJ4q+5BEqhZ3558frmXci0todmId/j3yQs5qrZdPpWooSceEh8xsLtArKLrV3RckJSqRKmLvwTzunbGIWQs3cEmnpjw0sCsn1dPrd1J1FJmEgvHiCq0JPv/dpkFARUrn8827GTlpPqu27OHuvqcz6pt6+VSqnuJcCWUQn3Cu8KejcMRTC5bbJyEukUrthYUbuOffn1C3ZnUmDTuPC09rHHVIIpEoMgm5e7viNGRmZ7r7p8cekkjldSivgF+/vJSn319D7JT46AfNG2r0A6m6StIxoSj/BHqUYXsilcqGHfsZ/ex8Fnyxg2G92jH2So1+IFKWSUg3s0WO4J0VW7hraiaH8gp49KYeXHWWXj4VgbIdgLT4s+MR79RgZrPNbEXwnXKEekOCOivMbEhQ1sDMMkOfrWb2cLBtqJltCW0bHmorP1Q+61hOVqQ4CgqcP89Zwff+/jFNTqjNrDsvUgISCSnLK6GSGgvMcffxZjY2WL8nXCHomfcAECOe5DLMbJa7bwe6heplADNCuz7n7ncmOOZ+d++WoFykzG3fe4gfTstk7mdbuLZ7K375nS7UqxXlj5xI+VOWPxGHSli/P3BxsDwRmMthSQjoC8wu7AZuZrOJj+Q9pbCCmXUkPpbdOyWOWCRJFq7bwR2T57Nl90F+9Z0u3HhuqkY/EEmg2EnIzBJ1OtgJrHX3PHc/v4THbubu2cHyRqBZgjqtgHWh9fVBWdgg4lc+4duB15lZb2A58EN3L2yjjpmlA3nAeHefWcKYRY7K3Zn00ReMe2EJTRrUZvqoCzi79UlRhyVSbpXkSuhR4r3fPiHeCaEL8CnQ0MxGJZrWwcxeB5onaOu+8Iq7u5mV6JlSyCDgltD6C8AUdz9oZrcTv8q6JNh2irtnmVl74A0zW+TuKxPEPQIYAZCamlrKsKSq2Xcoj5/OWMTMzA1cfHoTHr6hm0Y/EClCSZLQBmBY4btAZtYZeBD4CfHnMV9LQu5+2ZEaM7NNZtbC3bODCfI2J6iWxZe37ABaE79tV9hGV6CGu2eEjrktVP9J4HehbVnB96pgCKLuwNeSkLtPACYAxGKx0iZHqUJWbtnDqEkZrNi8h/+9vCOjv3WaRj8QKYaS9I7rGH4Z1d2XAJ3cfVUpjz0LGBIsDwHSEtR5FehjZilB77k+QVmhwYSeDwEECa1QP2BpUJ4STEmOmTUGLgKWlDJ2kf966ZNs+v3lXbbuOcQ//+c8xlzaQQlIpJhKciX0qZk9BkwN1m8AlgS/2HNLcezxwDQzGwasJT59OGYWA0a6+3B3zzGzccC8YJ8HDxurbiBw1WHtft/M+hF/7pMDDA3KzwCeMLMC4sl3fJBIRUolN7+A37y8jL+/t5oeqSfxyE09aNGwbtRhiVQo9tXn+UepaFYXuIMvR9F+j/hzogNAPXffk5QIIxaLxTw9PT3qMKScyd65nzufXUDG2u3celFb7r3yDGrV0OgHIoXMLMPdY0XVK8lUDvvN7C/En/048Jm7F14BVcoEJJLIe59v5ftTFnAgN5+/3tida85uGXVIIhVWSbpoX0y8p9ka4r3j2pjZEHd/OzmhiZQvBQXOo3M/56HZyzm1yQk8dnNPTmt6QtRhiVRoJXkm9Aegj7t/Bv99SXQK0DMZgYmUJzv2HeKHz2Xy5mdb6N+tJb/+zlnUr63RD0SOVUl+imoWJiAAd19uZjWTEJNIufLJ+h2MmjSfzbsPMG5AF24+T6MfiJSVkiShdDN7EpgUrN8E6Im9VFruzrMff8EvZsVHP/jXyAvp1kajH4iUpZIkoVHAaOD7wfo7xHvHiVQ6+w/lc9/zi5ixIItvdoyPfpBSX6MfiJS1kvSOOwg8FHxEKq1VW/YwatJ8lm/ezQ8v68iYSzT6gUiyFJmEzGwRR5kryN3PLtOIRCL0n0XZ3D39E2pWNybeei69OzaJOiSRSq04V0LXJD0KkYjl5hcw/j/LeOrd1XRrcxKP3tSDlidp9AORZCsyCbn72uI0ZGYfuPsFxx6SyPG1cecB7nx2PulrtzP0wrb89CqNfiByvJTliw51yrAtkePi/ZXx0Q/2Hcrnz4O706+rRj8QOZ7KMglpygOpMAoKnMfeWskfXvuM9k1OYOqIHpzWtEHUYYlUOXrlW6qcnfty+dG0TOYs20y/ri35zbUa/UAkKmX5k6c+rFLuLc7ayajJGWzceYBf9DuT711wikY/EIlQWSahW4quIhKdFxZu4O7pC0mpV4tpt19A99SUqEMSqfKK3QXIzHab2a7DPuvM7Hkza+/ui0tyYDNrZGazzWxF8J3wN4KZDQnqrDCzIUFZAzPLDH22mtnDoX0GmtkSM/vUzJ49WltS+RUUOH947TPGTFnAWa0a8sKYXkpAIuVESa6EHgbWA88Sv/U2CDgVmA/8Hbi4hMceC8xx9/FmNjZYvydcwcwaAQ8AMeIdHzLMbJa7bwe6heplADOC5Q7AvcBF7r7dzJoWoy2ppPYezONH0zJ59dNN3BBrw7gBXdT9WqQcKclPYz93f8Ldd7v7LnefAPR19+eA0vy3sj/x+YkIvgckqNMXmO3uOUGymA1cEa4QTCnRlPhYdgC3AY8UJhd331zctqRyWZezj+see5/ZSzbxwLc7M/66s5SARMqZklwJ7TOzgcD0YP164lN7Q+m6Zzdz9+xgeSPQLEGdVsC60Pr6oCxsEPCcfzlPeUcAM3sPqA783N1fKWZbUkl8vDqHkZMyyMsv4GkNvyNSbpUkCd0E/In4yNkOfAjcbGZ1gTsT7WBmrwPNE2y6L7zi7m5mpX3PaBBf7RRRA+hA/PZga+BtMzurJA2a2QhgBEBqamopw5KoTP34C36Wtpg2KfV4ckiM9k00+6lIeVWSUbRXAd8+wuZ3j7DPZUdqz8w2mVkLd882sxbA5gTVsvjqs6bWwNxQG12BGu6eEaqzHvjI3XOB1Wa2nHhSOmpbh8U9AZgAEIvF9BJuBZGXX8AvX1rK0++voXfHJvxlcHca1tW8iyLlWUl6x3U0szlmtjhYP9vM7j+GY88CCnuoDQHSEtR5FehjZilB77k+QVmhwcSnGA+bSZBszKwx8dtzq4rRllRgO/flcuvT83j6/TUM69WOvw+JKQGJVAAleUr7N+K9znIB3P0T4rfCSms8cLmZrQAuC9Yxs1gwgyvungOMA+YFnweDskID+XoSehXYZmZLgDeBu919WzHakgrq8817GPDoe3y4ahu/u+5sfnZNZ2pUVwcEkYrAvnyeX0RFs3nufo6ZLXD37kFZprt3K2rfiiwWi3l6umYxL6/mfraZMVMWULtGNR6/uSexto2iDklEiL864+6xouqVpGPCVjM7laAnnJldD2QffReR5HB3nnp3Nb9+eSmnNz+RJ4fEaKX5f0QqnJIkodHEH9Z3MrMsYDXxHnMix9XBvHzue34x0zPWc2WX5vxhYFfq1dIApCIVUUl+crOAfxB/ztII2EW8Q8GDSYhLJKEtuw8yclIGGWu3c9elHbjr0g5Uq6YBSEUqqpIkoTRgB/FhejYkJxyRI1uctZMRz6STs+8Qj9zYg6vPbhF1SCJyjEqShFq7u4a5kUi8vCib/522kJR6NZk+8kK6tGoYdUgiUgZKkoTeN7Oz3H1R0qIROUxBgfOnOSv405wV9Eg9iSduidGkQe2owxKRMlKSJNQLGGpmq4GDxEfSdnc/OymRSZW371Ae/zttIf9ZvJHre7bmV9/pQu0a1aMOS0TKUEmS0JVJi0LkMFk79nPbxHSWbdzF/VefwbBe7TQDqkglVJKx49YmMxCRQhlrc7j9nxkczC3gqaHn8K3Tm0YdkogkiV6ukHLlX+nruO/5xbQ8qQ5TR8Q4rWmDqEMSkSRSEpJyIb/A+c3LS3ny3dX0Oq0xf72xOyfVqxV1WCKSZEpCErmd+3P5/pQFvLV8C0MvbMv9V5+hAUhFqgglIYnUqi17GP5MOuty9vGba89i8LmaRFCkKlESksi8s2ILoyfPp0b1akwadh7ntT856pBE5DhTEpLjzt15+v01/PKlpXRoegJ/+16MNo3qRR2WiERASUiOq0N5Bfxf2mKmzlvH5Z2b8fAN3ahfW/8MRaqqyJ7+mlkjM5ttZiuC75Qj1BsS1FlhZkOCsgZmlhn6bDWzh0P7DDSzJWb2qZk9GyrPD+0zK/lnKWHb9hzk5ic/Yuq8ddz5rdN44uaeSkAiVVyUvwHGAnPcfbyZjQ3W7wlXMLNGwANAjPhkehlmNsvdtwPdQvUygBnBcgfi05Bf5O7bzSz8puP+yj4TbHm1NHsXwyems3XPQf48uDv9uraMOiQRKQei7AfbH5gYLE8EBiSo0xeY7e45QeKZDXxlJG8z6wg0Bd4Jim4DHgnq4+6bkxC7lMArizdy3WPvk1/g/GvkBUpAIvJfUSahZu5eOD34RqBZgjqtgHWh9fVBWdgg4Dm+Drf4AAAQz0lEQVR392C9I9DRzN4zsw/NLJy06phZelCeKOlJGXJ3/jJnBSMnZdChWQNm3XkRZ7c+KeqwRKQcSertODN7HWieYNN94RV3dzPzBPWKYxBwS2i9BtABuBhoDbwdTEGxAzjF3bPMrD3whpktcveVCeIeAYwASE3Veyulsf9QPndPX8iLn2Tzne6t+M21Z1GnpkbAFpGvSmoScvfLjrTNzDaZWQt3zzazFkCi22ZZxJNJodbA3FAbXYEa7p4RqrMe+Mjdc4HVZraceFKa5+5ZQVyrzGwu0B34WhJy9wnABIBYLFba5FhlZe/cz4hnMli8YSdjr+zE7b3bawRsEUkoyttxs4AhwfIQ4tOHH+5VoI+ZpQS95/oEZYUGA1MO22cmQeIys8bEb8+tCtqoHSq/CFhSNqciheZ/sZ1+f32P1Vv38uT3Yoz85qlKQCJyRFH2jhsPTDOzYcBaYCCAmcWAke4+3N1zzGwcMC/Y50F3zwm1MRC46rB2CxPXEiAfuNvdt5nZhcATZlZAPPmOd3cloTI0Y/56xs5YRPMT6zB5+Hl0bKYRsEXk6OzL5/mSSCwW8/T09KjDKNfyC5zfvbKMJ95exQXtT+bRm3qQUl8jYItUZWaW4e6xourpTUE5JrsP5HLX1EzeWLaZW84/hf/7dmdqagRsESkmJSEptTVb9zL8mXTWbN3LLwd04ebzT4k6JBGpYJSEpFTe/3wroybPxwyeGXYuF57aOOqQRKQCUhKSEvvnB2v4+QtLOLVJfZ783jmknqwRsEWkdJSEpNhy8wv4+axPmfzRF1x2RlP+eEM3GtSpGXVYIlKBKQlJseTsPcSoSRl8tDqHURefyo/7nE71anr/R0SOjZKQFOmzjbsZ/sw8Nu06yMM3dGNA98OH7xMRKR0lITmq2Us28YOpC6hfuwbTbr+Abm00AKmIlB0lIUnI3Xl07kp+/9pnnNWqIRNuidG8YZ2owxKRSkZJSL7mQG4+9/z7E9IyN9Cva0t+d/3ZGgFbRJJCSUi+YtOuA4x4Jp2F63dyd9/TueNiDUAqIsmjJCT/tXDdDkb8M509B/KYcEtP+pyZaCooEZGyoyQkAKRlZnH39E9o2qA2/77jQjo1PzHqkESkClASquIKCpzfv/YZj85dybntGvHYTT04+YTaUYclIlWEklAVtudgHj+YuoDXl27mxvNS+fm3z6RWDY2ALSLHT2S/ccyskZnNNrMVwXfKEeoNCeqsMLMhQVkDM8sMfbaa2cPBtj+Gypeb2Y6jtVVVfbFtH9c++h5vfraFB/ufya8GdFECEpHjLsorobHAHHcfb2Zjg/V7whXMrBHwABADHMgws1nuvh3oFqqXAcwAcPcfhsrHAN2L0VaV8sHKbdwxOYMCh2f+51wuOk0jYItINKL8r29/YGKwPBEYkKBOX2C2u+cEyWI2cEW4gpl1BJoC7yTYfzAwpbhtVQWTP1rLLU99RKP6tUgbfZESkIhEKsoroWbunh0sbwSaJajTClgXWl8flIUNAp7zw+YpN7NTgHbAGyVoq9LKzS9g3ItLeOaDtXzr9Cb8aXB3TtQI2CISsaQmITN7HUj0ssl94RV3dzPzBPWKYxBwyxHKp7t7fkkbNLMRwAiA1NTUUoZVfuzYd4g7Js/n/ZXbGNG7Pfdc0UkjYItIuZDUJOTulx1pm5ltMrMW7p5tZi2AzQmqZQEXh9ZbA3NDbXQFarh7RoJ9BwGji9vWYXFPACYAxGKx0ibHcmHFpt0Mfyad7B0H+P13u3J9z9ZRhyQi8l9RPhOaBRT2UBsCpCWo8yrQx8xSgt5zfYKyQuFnPv9lZp2AFOCDErRV6by5bDPfefR99h7MZ8qI85WARKTcifKZ0HhgmpkNA9YCAwHMLAaMdPfh7p5jZuOAecE+D7p7TqiNgcBVCdoeBEwNPycqRluVhrsz4e1VjH9lGWe2PJEJt8RoeVLdqMMSEfkaO+x5vhwmFot5enp61GEU24HcfH46YxEzFmRx9dkt+P31XalbSyNgi8jxZWYZ7h4rqp5GTKhENu86wIh/ZpC5bgc/urwjYy45TSNgi0i5piRUSSxav5Pbnkln5/5cHr+5B1d0aRF1SCIiRVISqgReWLiBu6cv5OT6tfn3qAvp3FIjYItIxaAkVIEVFDh/fH05f3njc85pm8JjN/eksUbAFpEKREmogtp7MI8fTcvk1U83cUOsDeM0AKmIVEBKQhXQupx93PZMOss37eb/runMrRe1VQcEEamQlIQqmI9X5zByUga5+QU8feu59O7YJOqQRERKTUmoApn68Rf8LG0xbVLq8eSQGO2bnBB1SCIix0RJqALIyy/gly8t5en319C7YxP+Mrg7DetqBGwRqfiUhMq5nftyuXPKfN5ZsZVhvdpx75WdqFFdHRBEpHJQEirHPt+8h9ueSWf99n387rqzGXhOm6hDEhEpU0pC5dTczzYzZsoCateoxpTbzifWtlHUIYmIlDkloXLG3Xnq3dX8+uWlnN78RJ4cEqOVRsAWkUpKSagcOZiXz33PL2Z6xnqu7NKcPwzsSr1a+isSkcpLv+HKiS27DzJyUgYZa7dz16UduOvSDlTTFNwiUskpCZUDi7N2MuKZdHL2HeKRG3tw9dkaAVtEqobI+vqaWSMzm21mK4LvlCPUGxLUWWFmQ4KyBmaWGfpsNbOHg21/DJUvN7MdobbyQ9tmHZ8zPbr/LMrmu49/gAPTR16oBCQiVUqUV0JjgTnuPt7Mxgbr94QrmFkj4AEgBjiQYWaz3H070C1ULwOYAeDuPwyVjwG6h5rc7+7dKAcKCpw/zVnBn+asoEfqSTx+S0+aNqgTdVgiIsdVlG899gcmBssTgQEJ6vQFZrt7TpB4ZgNXhCuYWUegKfBOgv0HA1PKLOIysu9QHqOfnc+f5qzguh6tmTLifCUgEamSorwSaubu2cHyRqBZgjqtgHWh9fVBWdgg4Dl393ChmZ0CtAPeCBXXMbN0IA8Y7+4zEwVmZiOAEQCpqanFO5tiytqxn9smprNs4y7uv/oMhvVqpxGwRaTKSmoSMrPXgeYJNt0XXnF3NzNPUK84BgG3HKF8urvnh8pOcfcsM2sPvGFmi9x95eE7uvsEYAJALBYrbVxfk7E2h9v/mcHB3AKeGnoO3zq9aVk1LSJSISU1Cbn7ZUfaZmabzKyFu2ebWQtgc4JqWcDFofXWwNxQG12BGu6ekWDfQcDow+LJCr5Xmdlc4s+LvpaEkuFf6eu47/nFtDypDlNHxDitaYPjcVgRkXItymdCs4AhwfIQIC1BnVeBPmaWEvSe6xOUFUr4zMfMOgEpwAehshQzqx0sNwYuApaUwXkcVX6B88sXl3D39E84t10jZo6+SAlIRCQQ5TOh8cA0MxsGrAUGAphZDBjp7sPdPcfMxgHzgn0edPecUBsDgasStD0ImHrYc6IzgCfMrIB48h3v7klNQjv35/L9KQt4a/kWhl7YlvuvPkMjYIuIhNhhz/PlMLFYzNPT00u839pte7n16Xl8sW0f4wZ0YfC5ZdvBQUSkPDOzDHePFVVPIyYkSe0a1aldozqTh5/Hee1PjjocEZFySUkoSZo3rMNLY3pp/DcRkaPQA4okUgISETk6JSEREYmMkpCIiERGSUhERCKjJCQiIpFREhIRkcgoCYmISGSUhEREJDIatqcIZraF+Nh2pdUY2FpG4USpspwH6FzKo8pyHqBzKXSKuzcpqpKSUJKZWXpxxk8q7yrLeYDOpTyqLOcBOpeS0u04ERGJjJKQiIhERkko+SZEHUAZqSznATqX8qiynAfoXEpEz4RERCQyuhISEZHIKAklgZnVMbOPzWyhmX1qZr+IOqZjZWbVzWyBmb0YdSzHwszWmNkiM8s0s5JPmVtOmNlJZjbdzJaZ2VIzuyDqmErDzE4P/i4KP7vM7AdRx1VaZvbD4Gd+sZlNMbM6UcdUGmZ2V3AOnyb770O345LAzAyo7+57zKwm8C5wl7t/GHFopWZmPwJiwInufk3U8ZSWma0BYu5eod/jMLOJwDvu/qSZ1QLqufuOqOM6FmZWHcgCznP3Y3k3LxJm1or4z3pnd99vZtOAl9396WgjKxkz6wJMBc4FDgGvACPd/fNkHE9XQkngcXuC1ZrBp8JmezNrDVwNPBl1LAJm1hDoDTwF4O6HKnoCClwKrKyICSikBlDXzGoA9YANEcdTGmcAH7n7PnfPA94Crk3WwZSEkiS4fZUJbAZmu/tHUcd0DB4GfgIURB1IGXDgNTPLMLMRUQdTSu2ALcA/glukT5pZ/aiDKgODgClRB1Fa7p4F/B74AsgGdrr7a9FGVSqLgW+Y2clmVg+4CmiTrIMpCSWJu+e7ezegNXBucIlb4ZjZNcBmd8+IOpYy0svdewBXAqPNrHfUAZVCDaAH8Ji7dwf2AmOjDenYBLcU+wH/ijqW0jKzFKA/8f8ktATqm9nN0UZVcu6+FPgt8BrxW3GZQH6yjqcklGTBbZI3gSuijqWULgL6Bc9SpgKXmNmkaEMqveB/q7j7ZuB54ve9K5r1wPrQ1fV04kmpIrsSmO/um6IO5BhcBqx29y3ungvMAC6MOKZScfen3L2nu/cGtgPLk3UsJaEkMLMmZnZSsFwXuBxYFm1UpePu97p7a3dvS/x2yRvuXuH+dwdgZvXNrEHhMtCH+K2HCsXdNwLrzOz0oOhSYEmEIZWFwVTgW3GBL4Dzzaxe0DnpUmBpxDGVipk1Db5TiT8PejZZx6qRrIaruBbAxKC3TzVgmrtX6K7NlUQz4Pn47wdqAM+6+yvRhlRqY4DJwW2sVcCtEcdTasF/CC4Hbo86lmPh7h+Z2XRgPpAHLKDijp7wbzM7GcgFRiez44u6aIuISGR0O05ERCKjJCQiIpFREhIRkcgoCYmISGSUhEREJDJKQiIiEhklIZFKIpimonEp9x1qZi3Loi2RklASEhGAocTHOxM5rpSERMqYmbUNJpt72syWm9lkM7vMzN4zsxVmdm7w+SAYBfv9wiF4gknR/h4snxVMLFbvCMc52cxeCyYeexKw0Labg4kVM83siWD0Dsxsj5n9MdhnTjDE1PXE54qaHNSvGzQzxszmB5MAdkrmn5lUXUpCIslxGvAHoFPwuRHoBfwY+CnxsQS/EYyC/X/Ar4P9/gScZmbfAf4B3O7u+45wjAeAd939TOKDsaYCmNkZwA3ARcFI7vnATcE+9YH0YJ+3gAfcfTqQDtzk7t3cfX9Qd2sw4vhjQdwiZU5jx4kkx2p3XwRgZp8Cc9zdzWwR0BZoSHx8wQ7E5ziqCeDuBWY2FPgEeMLd3zvKMXoTTDbm7i+Z2fag/FKgJzAvGCevLvF5rSA+J9RzwfIk4iM9H0nhtgySOKmZVG1KQiLJcTC0XBBaLyD+czcOeNPdv2NmbYG5ofodgD2U/hmNARPd/d5i1D3a4JGFMeej3xWSJLodJxKNhkBWsDy0sDCYuvvPxK9yTg6e1xzJ28Rv82FmVwIpQfkc4PrQcPyNzOyUYFs1oLDNG4F3g+XdQINjOB+RUlESEonG74DfmNkCvnqV8UfgEXdfDgwDxhcmkwR+AfQObvddS3w+G9x9CXA/8WnMPwFmE59eBOKzsJ5rZouBS4AHg/KngccP65ggknSaykGkCjGzPe5+QtRxiBTSlZCIiERGV0Ii5ZyZ3QrcdVjxe+4+Oop4RMqSkpCIiERGt+NERCQySkIiIhIZJSEREYmMkpCIiERGSUhERCLz/wFuvgtMkmSBKAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "plt.plot(max_depth_s, test_means)\n",
    "plt.xlabel( 'max_depth' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 3 candidates, totalling 15 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=5)]: Using backend LokyBackend with 5 concurrent workers.\n",
      "[Parallel(n_jobs=5)]: Done  10 out of  15 | elapsed:  1.9min remaining:   58.4s\n",
      "[Parallel(n_jobs=5)]: Done  15 out of  15 | elapsed:  2.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=7, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.0001, max_depth=-1, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "    ....0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid='warn', n_jobs=5,\n",
       "       param_grid={'max_depth': range(9, 15, 2)}, pre_dispatch='2*n_jobs',\n",
       "       refit=False, return_train_score='warn', scoring='neg_log_loss',\n",
       "       verbose=5)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'n_estimators':628,\n",
    "          'num_leaves': 70,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':1.0,\n",
    "          'subsample':1.0,\n",
    "          'learning_rate':0.0001\n",
    "         }\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "\n",
    "max_depth_s = range(9,15,2) \n",
    "\n",
    "tuned_parameters = dict( max_depth = max_depth_s)\n",
    "\n",
    "grid_search = GridSearchCV(lg, n_jobs=5, param_grid=tuned_parameters, cv = kfold, scoring=\"neg_log_loss\", verbose=5, refit = False)\n",
    "grid_search.fit(X_train , Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.07332661392213395\n",
      "{'max_depth': 13}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAELCAYAAACMOhKAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VeWdx/HPD8JOwr4kIJuCrAoSgvtYS5FqW5dRBBfCYimtdneqHe3Y0Wlrnal1Wq1KK5uKW9VqF6tIa9W6QMIiu+xCCGuAhCVk+80f9wQumQRyITfnJvm+X6/7uuc89znn/O6B5JfnPM95jrk7IiIiiaZR2AGIiIhURglKREQSkhKUiIgkJCUoERFJSEpQIiKSkJSgREQkISlBiYhIQlKCEhGRhKQEJSIiCSkp7ADqso4dO3qvXr3CDkNEpM7Izs7e7e6dqlNXCeo09OrVi6ysrLDDEBGpM8xsc3Xr6hKfiIgkJCUoERFJSEpQIiKSkJSgREQkISlBiYhIQlKCEhGRhKQEJSIiCUn3QYmIyEm5O9vzC1m5LZ+dBUcYn9Ej7sdUghIRkeOUljkbdx9kxbb9rNyWz8rcfFZsyyfvYBEAzZs04obh3UlqHN+LcEpQIiINWGFxKWu2F7BiWz4rc/ezYls+q3MLOFxcCkDTxo3o17U1XxjQhYFpKQxKS6F/akrckxMoQYmINBj7DhUd1yJasW0/63cdpLTMAUhunsTA1BTGZZzBoLQ2DEpL4cxOrWmaFM5wBSUoEZF6xt3Ztj/SX7RiW6RVtHJbPjn7Dh+t0zWlOQPTUrhiUFcGpaUwMLUNZ7RvgZmFGPnxlKBEROqwktKyoL8okoxW5kaS0d5DxQCYQe+OrTivZztuvaAnA1NTGJiWQsfWzUKO/OSUoERE6ojDRaWs3p4f9BdF3tdsz6ewuAyApkmN6N81mTGDuwaJqA0DUpNp2bRu/qqvm1GLiNRzew8WBUno2CW69bsOEHQXkdI8iUFpbbh5ZE8GpaUwKK0NfTq1okktDF6oLUpQIiIhcndy9h0OLtFFEtHKbfvZtr/waJ3UNs0ZlJbCF4ekBv1FKXRvl1j9RfGgBCUiUktKSstYv+vY/UXll+r2H470FzUy6NOpNSN6t2dgaqRVNDAthfatmoYceTiUoERE4uBQUQmrcguCQQvB/UXbCygqifQXNUtqRP/UFK4MWkWD0lLo3zWFFk0bhxx54lCCEhE5TXsOHIm6tyiSkDbsPogH/UVtWjRhUFoKmRf0PNoq6tOxVa3c7FqXKUGJiFSTu7N17+HjBi6s2JbP9vxj/UXd2rZgYFoKXz43LXKZrlsb0to0r/f9RfGgBCUiUoni0jLW7TwQlYgi9xgVFJYAkf6iszq35oIzOwT9RZH7i9q2bJj9RfGgBCUiDd7BIyWsyg3uLcrJZ0Xufj7dfoCi0kh/UfMmjRiQmsJXzk07OgXQ2V2Tad5E/UXxpAQlIg3KroIjx91ftGpbPhv3HOsvateyCYPS2jDpol5HJ0ft3bE1jRvpEl1tCy1BmVl74AWgF7AJGOvueyuplwncG6z+l7vPNrNk4L2oat2BZ9z9O2Y2DbgdKAUOAFPdfaWZZQDTy3cL/NjdXw2OsQkoCLYpcff0mvyuIlL7ysqcLXsPHZsCKOgv2llw5Gid7u1aMCgthWuGdQv6i1LomqL+okRhXv5nQ20f2OwhIM/dHzSzu4F27n5XhTrtgSwgHXAgGxheMZGZWTbwXXd/18xS3D0/KP8K8A13H2NmLYEidy8xs1RgKZAWrG8C0t19dyzfIT093bOysk7l64tIDSoqKWPtzoKoG13zWZWbT8GRSH9R40ZG386tj85DVz6Srk2LJiFH3vCYWXZ1GwFhXuK7GrgsWJ4NvAPcVaHOFcA8d88DMLN5wBjgufIKZtYP6EzQoipPToFWRBIb7n4oqrx5ebmI1C0FhcWs3l7Aipz9R290/XRHAcWlkR/plk0bMyA10ioqH7jQr4v6i+qiMBNUF3fPDZa3A10qqdMN2BK1vjUoizYOeMGjmoJmdjvwPaApcHlU+UhgBtATuNXdS4KPHHjLzBx40t3LLwWKSIh2FhQe1ypasW0/m/Yc+1uzQ6umDExLYcrFfY72F/Xq0Er9RfVEXBOUmb0NdK3ko3uiV9zdg+RwKsYBt1bY32PAY2Z2E5H+q8yg/GNgkJkNAGab2RvuXghc7O45ZtYZmGdmq9393Sq+01RgKkCPHj1OMWQRiVZW5mzOO3RcX9GKbfnsPnCsv6hH+5YMSkvhX8/rzqBukct0nZObqb+oHotrgnL3UVV9ZmY7zCzV3XODPqGdlVTL4dhlQIgMhngnah/nAknunl3FYZ4HHq8krlVmdgAYDGS5e05QvtPMXgUygEoTVNC6mg6RPqiqvp+IVO5ISSlrdxw47mF6q3LzOVgUecR4UiOjb5dk/qVfp6NTAA1ISyGlufqLGpowL/G9TqRl82Dw/loldd4Efmpm7YL10cAPoz4fT1R/FICZ9XX3tcHqVcDaoLw3sCUYFNET6A9sMrNWQCN3LwiWRwP318QXFGno8guLWRXVIlqZm8/aHQWUBM+MaBX0F10/vPvRgQt9u7SmWZL6iyTcBPUg8KKZTQE2A2MBzCwdmObut7l7npk9ACwMtrm/fMBEYCxwZYX93mFmo4BiYC/B5T3gYuBuMysGyoiM7tttZn2AV4PLBEnAXHf/a01/WZH6zN3ZWXAk0iLKOfYwvc/yjvUXdWzdjEFpKXzu7E5HR9L1bN+SRuovkiqENsy8PtAwc2mIysqcjXsOHj8F0LZ89hwsOlqnV4eWR1tE5YMXOic3DzFqSRR1ZZi5iCS4wuJSPt1REDVwYT+rtxdwKOgvatLY6Nclmcv7d470F3VrQ/+uySSrv0hqgBKUiACw/3DxcS2ilbn5rNt54Gh/UetmSQxMTWFs+hlH7y/q2zmZpkl6ZITEhxKUSAPj7mzPL4zqK4qMpNu69/DROp2TI/1FowZ0OXqJ7ox26i+S2qUEJVKPlZY5G3cfOO7ZRStz88kL+ovMoHeHVpx7RltuGtkj0m+UmkKn5GYhRy6iBCVS7yzclMcfFuewMjef1bkFHC6O9Bc1bdyIfl1b84UBXRjULYWBqSn0T02hdTP9GpDEpP+ZIvXIRxv2MOGpBTRLasTAtBTGZ/Q4eonurM6taaJHjEsdogQlUk+sys3nq7Oz6NGhJb+fdoGe7Cp1nv6cEqkHtu49xMSZC2jVLInZkzOUnKReUIISqePyDhYxYcYCDheVMntyBt3atgg7JJEaoUt8InXYoaISJs9ayNa9h3lmykjO7pocdkgiNUYtKJE6qri0jNufXcQnW/fx6/HDyOjdPuyQRGqUWlAidZC7c/fLy/j7ml389NohXDGosseuidRtakGJ1EEPvbmGlxdt5Tuj+nLTSD04U+onJSiROmbmPzfy+DvruWlkD779+b5hhyMSN0pQInXIH5du4/4/rWT0wC48cPVgPe5c6jUlKJE64oN1u/n+i0sZ0bM9vxo/jMaauFXqOSUokTpgec5+pj6dTe+OrfjthHSaN9Ej0aX+U4ISSXCf7TnExJkLSWmexKzJI2jTUg8DlIZBCUokge0+cIQJMz6mpKyMOVMySG2jWSKk4VCCEklQB49EZonYnl/IU5kjOKuzZomQhkU36ookoKKSMqY9k82KbflMv3U4w3u2CzskkVqnFpRIgikrc37w+6W8t3Y3P7tuCJ8f0CXskERCoQQlkmB+9sYq/rBkG/92xdmMTT8j7HBEQqMEJZJAfvvuBn773kYmXNCTb1x2ZtjhiIRKCUokQfxhcQ4/+csqrhzSlfu+PEizREiDpwQlkgDe/XQXd760lPP7tOfhsUM1S4QISlAioftk6z6mPZNN3y7JTNcsESJHKUGJhGjj7oNMmrmQ9q2aMnvSCFKaa5YIkXJKUCIh2VlQyIQZH+PAnMkZdE5pHnZIIglFCUokBAWFxUycsZDdBUXMmDiCPp1ahx2SSMJRghKpZUdKSvna09l8uqOAx285j6FntA07JJGEFFqCMrP2ZjbPzNYG75XO5WJmmUGdtWaWGZQlm9mSqNduM3sk+GyamS0Lyt83s4EV9tfDzA6Y2Z1RZWPMbI2ZrTOzu+P5vaVhKytzvv/iUj5Yv4eHrj+Hy87uHHZIIgkrzBbU3cB8d+8LzA/Wj2Nm7YH7gJFABnCfmbVz9wJ3H1r+AjYDrwSbzXX3IUH5Q8DDFXb7MPBG1DEaA48BXwQGAuMrJjWRmuDuPPDnlfzpk1zu/mJ/rjuve9ghiSS0MBPU1cDsYHk2cE0lda4A5rl7nrvvBeYBY6IrmFk/oDPwHoC750d93ArwqLrXABuBFVF1MoB17r7B3YuA54PYRGrUE//YwMx/bmLyRb352qV9wg5HJOGFmaC6uHtusLwdqGxGzG7Alqj1rUFZtHHAC+4enYhuN7P1RFpQ3wrKWgN3Af95CscQOS2/z97Kz/+6mq+cm8a9Vw3QLBEi1RDXx22Y2dtA10o+uid6xd3dzLySetUxDri1wv4eAx4zs5uAe4FM4MfAL939wOn8cjCzqcBUgB49epzyfqTh+Pvqndz18idcfFZH/ueGc2mkWSJEqiWuCcrdR1X1mZntMLNUd881s1RgZyXVcoDLota7A+9E7eNcIMnds6s4zPPA48HySOB6M3sIaAuUmVkhkA1ETxndPThuVd9pOjAdID09/VSTqjQQiz/byzeeXcSA1GSeuHU4TZM0cFakusL8aXmdSMuG4P21Suq8CYw2s3bBKL/RQVm58cBz0RuYWd+o1auAtQDufom793L3XsAjwE/d/VFgIdDXzHqbWVMiLbLXT/fLiazfdYDJsxbSOaUZMydm0LqZng8qEoswf2IeBF40sylERuGNBTCzdGCau9/m7nlm9gCRJAJwv7vnRe1jLHBlhf3eYWajgGJgL8eSYKXcvcTM7iCS+BoDM9x9xYm2ETmZHfmFTHhqAY0bGXMmZ9ApuVnYIYnUORY1tkBilJ6e7llZWWGHIQlm/+FibnzyQ7bkHeL5qRcwpHubsEMSSRhmlu3u6dWpq2sOIjWosLiUqXOyWL/rADMmjlByEjkNSlAiNaS0zPnuC0v4eGMe/ztuKJf07RR2SCJ1moYUidQAd+fHr6/gjeXbufeqAVw9VLfSiZwuJSiRGvDo39bx9Eeb+dqlfbjtEs0SIVITlKBETtPzCz7jF/M+5bph3bhrTP+wwxGpN5SgRE7DvJU7+PdXl/Ev/Trx8+vP0SwRIjWo2gnKzB4ysxQza2Jm881sl5ndEs/gRBJZ1qY87pi7iCHd2vCbm8+jSWP9vSdSk2L5iRodzBT+JWATcBbwb/EISiTRrd1RwJTZWaS1bcGMiSNopVkiRGpcLAmq/CfwKuAld98fh3hEEl7u/sNMmLGApkmNmDM5gw6tNUuESDzE8mffn8xsNXAY+LqZdQIK4xOWSGLaf6iYzBkLKCgs4YWvnc8Z7VuGHZJIvVXtFpS73w1cCKS7ezFwED3YTxqQwuJSbpuzkE27DzF9wnAGpWmWCJF4imWQxA1AsbuXmtm9wDNAWtwiE0kgJaVlfPO5xWRt3ssvbxzKhWd2DDskkXovlj6oH7l7gZldDIwCnuLYs5ZE6i1350evLWfeyh38+MuDuOqc1LBDEmkQYklQpcH7VcB0d/8z0LTmQxJJLL98ey3PLdjC7Z87k8wLe4UdjkiDEUuCyjGzJ4Ebgb+YWbMYtxepc575aDO/mr+WsenduXP02WGHI9KgxJJgxhJ5qN8V7r4PaI/ug5J67K/Lc/nRa8v5fP/O/PTaIZhplgiR2hTLKL5DwHrgiuAJtJ3d/a24RSYSoo837OFbzy9h6BltefSm80jSLBEitS6WUXzfBp4FOgevZ8zsm/EKTCQsq7fnc9ucLM5o14IZmSNo0bRx2CGJNEix3Kg7BRjp7gcBzOznwIfAr+MRmEgYtu49ROaMBbRs2pg5U0bSrpXGAYmEJZYEZRwbyUewrIvyUm/sPVjEhBkLOFRUykvTLqBb2xZhhyTSoMWSoGYCH5vZq8H6NUTuhRKp8w4VlTB59kK27j3M05Mz6N81JeyQRBq8aicod3/YzN4BLg6KJrn74rhEJVKLikvLuGPuYpZu2cdvbh7OyD4dwg5JRKhGgjKz9lGrm4LX0c/cPa/mwxKpHe7Ov7+yjL+t3slPrh3MmMFdww5JRALVaUFlA86x/iYP3i1Y7hOHuERqxX+/uYaXsrfy7c/35eaRPcMOR0SinDRBuXvv6uzIzAa5+4rTD0mkdsz650Z+8856xmf04Duj+oYdjohUUJN3Hz5dg/sSias/fbKN//zTSr4wsAsPXD1Is0SIJKCaTFD6CZc64YP1u/neC0tJ79mOX48fplkiRBJUTf5k+smriIRrxbb9TJ2TTa+OLfndhBE0b6JZIkQSlf50lAZjS94hJs5cSErzJGZPzqBNyyZhhyQiJ1CTCaqoBvclUqP2HDjChBkLKCopY86UDFLbaJYIkURX7Rt1zey8Sor3A5vdvcTdz6+5sERqzsEjJUyetZDc/Yd59raRnNU5OeyQRKQaYmlB/Qb4CJgO/JbIRLEvAWvMbHSsBzaz9mY2z8zWBu/tqqiXGdRZa2aZQVmymS2Jeu02s0eCz6aZ2bKg/H0zG1hhfz3M7ICZ3RlVtilqm6xYv4skrqKSMr7+7CKWb8vn0fHnMbxn+5NvJCIJIZYEtQ0Y5u7p7j4cGAZsAL4APHQKx74bmO/ufYH5wfpxglks7gNGAhnAfWbWzt0L3H1o+QvYDLwSbDbX3YcE5Q8BD1fY7cPAG5XE87lgf+mn8F0kAZWVOXe9/AnvfrqLn147mFEDu4QdkojEIJYE1S/6Rlx3Xwn0d/cNp3jsq4HZwfJsIpPPVnQFMM/d89x9LzAPGBNdwcz6EXk+1XtBXPlRH7cianShmV0DbAR0Q3ED8PO/rubVxTl8/wv9uHFEj7DDEZEYxTKb+Qozexx4Pli/EVhpZs2A4lM4dhd3zw2WtwOV/XnbDdgStb41KIs2DnjB3aMT0e3A94CmwOVBWWvgLiItvjsr7MOBt8zMgSfdffopfB9JIL97bwNPvruBCRf05I7Lzwo7HBE5BbEkqInAN4DvBOv/JPKLvhj4XGUbmNnbQGWzb94TveLuHiSHUzEOuLXC/h4DHjOzm4B7gUzgx8Av3f1AJbMGXOzuOWbWGZhnZqvd/d3KDmZmU4GpAD166K/yRPTakhz+68+ruHJIV+77smaJEKmrYnncxmEz+zXwFpEWxxp3L285Hahim1FV7c/MdphZqrvnmlkqsLOSajnAZVHr3YF3ovZxLpDk7tlVHOZ54PFgeSRwvZk9BLQFysys0N0fdfecIN6dwfOuMoBKE1TQupoOkJ6erpuTE8x7a3dx50tLGdm7PQ+PHUrjRkpOInVVtfugzOwyYC3wKJERfZ+a2aWncezXibRsCN5fq6TOm8BoM2sXjPIbHZSVGw88VyHO6Fk/rwpixt0vcfde7t4LeAT4qbs/amatzCw52LZVcIzlp/G9JCTLtu5n2tPZnNmpNb/NTNcsESJ1XCyX+H4BjHb3NXB0cMJzwPBTPPaDwItmNoXIKLyxwX7TgWnufpu755nZA8DCYJv7Kzx/aixwZYX93mFmo4hcetzLsSRYlS7Aq8FloCQiowD/eorfSUKyafdBJs5cQNuWTZk9OYOU5polQqSus6ixBSeuaPaJu59zsrKGJD093bOydNtU2HYWFHL94x9SUFjM779+IWd2ah12SCJSBTPLru7tPLG0oLLM7HfAM8H6zYB+O0uoDhwpYdLMhewqOMLcr45UchKpR2JJUF8Hbge+Fay/R6QvSiQURSVlTHs6m9XbC/hdZjrDelQ6GYmI1FGxjOI7QmQWhoozM4jUurIy586XlvL+ut38zw3n8rmzO4cdkojUsJMmKDNbxgme9dSQ+6AkHO7Of/15Fa8v3cZdY/pz/fDuYYckInFQnRbUl+IehUgMnnx3AzP+uZFJF/Vi2r/0CTscEYmTkyYod99cnR2Z2YfufsHphyRStZezt/LgG6v58rlp/OiqgZolQqQeq8kHFjavwX2J/D9/X7OTH7z8CRed1YH/ueEcGmmWCJF6rSYTlKb9kbhZ/NlevvHMIvp3TeaJW4bTLEmzRIjUdzWZoETiYv2uA0yetZBOyc2YNSmDZM0SIdIg1GSC0vUWqXE78guZ8NQCGpkxZ3IGnZKbhR2SiNSSWG7UPZlbT15FpPryC4uZOHMhew8V8fzU8+nVsVXYIYlILap2gjKzAv5/P9N+ItMdfd/dNQO41JjC4lKmzsli7Y4CZkwcwTnd24YdkojUslhaUI8QeaLtXCKX88YBZwKLgBkc/9wmkVNWWuZ878UlfLQhj0duHMql/TqFHZKIhCCWPqivuPuT7l7g7vnBg/uucPcXAE2CJjXC3fnPP67gL8u2c+9VA7hmWLewQxKRkMSSoA6Z2VgzaxS8xgKFwWcaYi414rG/r2POh5uZemkfbrtEs0SINGSxJKibiQyE2AnsCJZvMbMWwB1xiE0amBcWfsb/vPUp1w7rxt1j+ocdjoiELJbZzDcAX67i4/drJhxpqN5euYMfvrKMS/t14qHrNUuEiMTQgjKzfmY238yWB+vnmNm98QtNGorszXncPncRg7u14fGbz6NJY90/LiKxXeL7LfBDoBjA3T8hMpJP5JSt21nAlNlZpLZpzoyJI2jVrCZvzRORuiyWBNXS3RdUKCupyWCkYcndf5gJTy0gqVEj5kweScfWmiVCRI6JJUHtNrMzCUbsmdn1QG5copJ6b/+hYjJnLCC/sIRZk0bQo0PLsEMSkQQTy/WU24HpQH8zywE2EhnZJxKTwuJSbpuzkE27DzFr0ggGd2sTdkgikoBiSVA5wEzg70B7IB/IBO6PQ1xST5WUlvGt5xaTtXkvvx4/jAvP6hh2SCKSoGJJUK8B+4hMbbQtPuFIfebu/Oi1Fby1cgc//vJAvnROWtghiUgCiyVBdXf3MXGLROq9R95ey3MLPuMbl53JxIt6hx2OiCS4WAZJfGBmQ+IWidRrz368mf+dv5Ybhnfn3644O+xwRKQOiKUFdTEw0cw2AkeIzGju7n5OXCKTeuPNFdv50R+Wc3n/zvzsuiGYaZYIETm5WBLUF+MWhdRbCzbm8c3nFnNO97Y8etMwkjRLhIhUUyxz8W2OZyBS/6zZXsBtsxfSvV0LZkwcQcummiVCRKpPf85KXOTsO0zmjAW0aNqYOZMzaN+qadghiUgdoz9ppcbtPVjEhKc+5mBRCS9Nu4Du7TRLhIjELrQWlJm1N7N5ZrY2eK/0qbxmlhnUWWtmmUFZspktiXrtNrNHgs+mmdmyoPx9MxsYlPcys8NR2zwRdYzhwTbrzOxXpl78U3a4qJTJsxeyZe9hfjchnf5dU8IOSUTqqDAv8d0NzHf3vsD8YP04ZtYeuA8YCWQA95lZu+Cx80PLX8Bm4JVgs7nuPiQofwh4OGqX66O2mxZV/jjwVaBv8NL9XqegpLSMO+YuYumWffxq3FBG9ukQdkgiUoeFmaCuBmYHy7OBayqpcwUwz93z3H0vMI8KycPM+gGdgfcA3D0/6uNWnORx9GaWCqS4+0fu7sCcKmKRE3B3/v3VZcxfvZP7rx7MmMGpYYckInVcmH1QXdy9fDb07UCXSup0A7ZErW8NyqKNA14IkgsAZnY78D2gKXB5VN3eZraYyDyC97r7e8H+tp7kGHISv3jrU17M2sq3Pt+XW87vGXY4IlIPxDVBmdnbQNdKPronesXd3cxO2NI5gXHArRX29xjwmJndBNxLZFLbXKCHu+8xs+HAH8xsUKwHM7OpwFSAHj16nGLI9cvsDzbx6N/XMT7jDL47qm/Y4YhIPRHXBOXuo6r6zMx2mFmqu+cGl9l2VlItB7gsar078E7UPs4Fktw9u4rDPE+kfwl3P0JkBgzcPdvM1gP9gmN0r3CMnBN8p+lEHjtCenr6qSbVeuPPn+Ty4z+uYNSALjxw9WDNEiEiNSbMPqjXibRsCN5fq6TOm8BoM2sXjPIbHZSVGw88F72BmUX/CX8VsDYo72RmjYPlPkQGQ2wILjPmm9n5wei9CVXEIhV8sH43331hCcN7tNMsESJS48Lsg3oQeNHMphAZhTcWwMzSgWnufpu755nZA8DCYJv73T0vah9jgSsr7PcOMxsFFAN7OZYELwXuN7NioCw4Rvm+vgHMAloAbwQvOYGV2/L52pxsenZoye8y02nepHHYIYlIPWNRYwskRunp6Z6VlRV2GLVuS94hrnv8A5IaGS9//ULS2rYIOyQRqSPMLNvd06tTV9dkJCZ7DhxhwowFFJWUMWdyhpKTiMSNEpRU28EjJUyetZBt+w4zY2I6fbskhx2SiNRjSlBSLcWlZdw+dxHLcvbz6E3nMbxn+7BDEpF6TpPFykm5O3e9/AnvrNnFg9cN4QsDK7unWkSkZqkFJSf14F9X88qiHL73hX6My9DNySJSO5Sg5ISeen8jT/5jA7ee35NvXn5W2OGISAOiBCVVem1JDg/8aSVfHNyVH39lkGaJEJFapQQllXpv7S7ufGkpI3u355c3DqVxIyUnEaldSlDy/yzbup9pT2dzZqfWTJ+gWSJEJBxKUHKczXsOMmnWAtq2bMrsyRm0adEk7JBEpIFSgpKjdhVEZokoLXPmTMmgS0rzsEMSkQZM90EJAAeOlDBp1gJ25h9h7ldHcman1mGHJCINnBKUUFRSxrSns1mVW8BvJwxnWI92YYckIqJLfA1dWZlz50tLeX/dbh68bgiX99csESKSGJSgGjB35yd/WcXrS7fxgzFnc0P6GWGHJCJylBJUAzb93Q089f5GJl7Yi6//y5lhhyMichwlqAbqlUVb+dkbq/nSOan8x5cGapYIEUk4SlAN0DtrdvKD33/CRWd14Bdjz6WRZokQkQSkBNXALNmyj68/s4izuybzxC3DaZakWSJEJDEpQTUgG3YdYPKshXRMbsrMSSNIbq5ZIkQkcSlBNRA78wuZMGMBBsyZPJLOyZolQkQSm27UbQDyC4t9L/IvAAANFUlEQVTJnLmQvINFPD/1fHp3bBV2SCIiJ6UWVD13pKSUqXOyWLujgCduGc453duGHZKISLWoBVWPlZY533thKR9tyOORG4dyab9OYYckIlJtakHVU+7O/X9cwZ+X5XLPlQO4Zli3sEMSEYmJElQ99Zt31jP7w8189ZLefPXSPmGHIyISMyWoeujFrC3895truGZoGj/84oCwwxEROSVKUPXM/FU7+OEry7ikb0ceul6zRIhI3aUEVY9kb97L7XMXMSgthSduGU7TJP3zikjdpd9g9cS6nQVMmb2QrinNmTFxBK2aaYCmiNRtSlD1wPb9hUx4agFJjRoxZ/JIOrZuFnZIIiKnLbQEZWbtzWyema0N3it9zriZZQZ11ppZZlCWbGZLol67zeyR4LNpZrYsKH/fzAYG5b3M7HDUNk9EHeMdM1sT9Vnn2jgHNWH/oWIyZywgv7CEWZNG0KNDy7BDEhGpEWFeB7obmO/uD5rZ3cH6XdEVzKw9cB+QDjiQbWavu/teYGhUvWzglWB1rrs/EZR/BXgYGBN8tt7dj25Xwc3unlUzX612FBaX8tU5WWzYfYDZkzIY3K1N2CGJiNSYMC/xXQ3MDpZnA9dUUucKYJ675wVJaR7Hkg0AZtYP6Ay8B+Du+VEftyKS2Oqd0jLn288vZuHmPB4eO5QLz+oYdkgiIjUqzATVxd1zg+XtQJdK6nQDtkStbw3Koo0DXnD3o4nIzG43s/XAQ8C3our2NrPFZvYPM7ukwn5mBpf3fmQJ/nhZd+dHry3nzRU7+I8vDeTL56aFHZKISI2L6yU+M3sb6FrJR/dEr7i7m9mptnTGAbdW2N9jwGNmdhNwL5AJ5AI93H2PmQ0H/mBmg4IW183unmNmycDLwf7mVPGdpgJTAXr06HGKIZ+eX81fx9yPP+Prl53JpIt6hxKDiEi8xbUF5e6j3H1wJa/XgB1mlgoQvO+sZBc5wBlR692DMoLtzgWS3D27ihCeJ7h06O5H3H1PsJwNrAf6Bes5wXsBMBfIOMF3mu7u6e6e3qlT7U++Ovfjz/jl25/yr+d15wdXnF3rxxcRqS1hXuJ7nUjLhuD9tUrqvAmMNrN2wSi/0UFZufHAc9EbmFnfqNWrgLVBeSczaxws9wH6AhvMLMnMOgblTYAvActP87vFxZsrtnPvH5bxubM78eC/DiHBr0SKiJyWMEfxPQi8aGZTgM3AWAAzSwemuftt7p5nZg8AC4Nt7nf3vKh9jAWurLDfO8xsFFAM7OVYErwUuN/MioGy4Bh5ZtYKeDNITo2Bt4Hf1vSXPV0LN+XxrecWc073tjx283k0aaxb2ESkfrOosQUSo/T0dM/Kiv/I9DXbC7jhiQ/omNyM30+7kPatmsb9mCIi8WBm2e6eXp26+jM8weXsO0zmjAU0b9KYOZMzlJxEpMHQhG0JbN+hIjJnLOBgUQkvfu0CurfTLBEi0nAoQSWow0WlTJ61kM/yDjFncgYDUlPCDklEpFbpEl8CKikt45vPLWLxln38741DOb9Ph7BDEhGpdUpQCcbduefV5by9aif3Xz2YLw5JDTskEZFQKEElmIfnfcoLWVv45uVncev5PcMOR0QkNEpQCeTpDzfx67+tY9yIM/jeF/qFHY6ISKiUoBLEX5bl8h+vr2DUgC781zWDNUuEiDR4SlAJ4MP1e/jO80s4r0c7fj1+GEmaJUJERAkqbCu35TN1ThY9O7Tkqcx0WjRtHHZIIiIJQQkqRFvyDpE5cwGtmycxe3IGbVtqlggRkXJKUCHJOxiZJeJIcSmzJ2eQ1rZF2CGJiCQUzSQRgkNFJUyatZCcfYd59raR9OuSHHZIIiIJRy2oWlZcWsY3nl3Esq37+PX4YaT3ah92SCIiCUkJqpaVlEYeb/KTa4cwelDXkKMREUlcusRXy1o0bcyMzBE0aqT7nERETkQtqBAoOYmInJwSlIiIJCQlKBERSUhKUCIikpCUoEREJCEpQYmISEJSghIRkYSkBCUiIgnJ3D3sGOosM9sFbD7FzTsCu2swnJqiuGKjuGKjuGJTH+Pq6e6dqlNRCSokZpbl7ulhx1GR4oqN4oqN4opNQ49Ll/hERCQhKUGJiEhCUoIKz/SwA6iC4oqN4oqN4opNg45LfVAiIpKQ1IISEZGEpAQVZ2b2bTNbbmYrzOw7lXxuZvYrM1tnZp+Y2XkJENNlZrbfzJYEr/+IYywzzGynmS2PKmtvZvPMbG3w3q6KbTODOmvNLDOB4iqNOnev10JcNwT/lmVmVuXIKjMbY2Zrgv9rdydQXJvMbFlwvrJqIa7/NrPVwc/bq2bWtopta/t8VTeu2j5fDwQxLTGzt8wsrYpta/7n0d31itMLGAwsB1oSeTjk28BZFepcCbwBGHA+8HECxHQZ8KdaOkeXAucBy6PKHgLuDpbvBn5eyXbtgQ3Be7tguV3YcQWfHajl8zUAOBt4B0ivYrvGwHqgD9AUWAoMDDuuoN4moGMtnq/RQFKw/PMq/n+Fcb5OGldI5yslavlbwBOVbBeXn0e1oOJrAJGEc8jdS4B/ANdVqHM1MMcjPgLamllqyDHVGnd/F8irUHw1MDtYng1cU8mmVwDz3D3P3fcC84AxCRBXXFUWl7uvcvc1J9k0A1jn7hvcvQh4nsj3CTuuuKoirreC//sAHwHdK9k0jPNVnbjiqoq48qNWWwGVDVyIy8+jElR8LQcuMbMOZtaSSGvpjAp1ugFbota3BmVhxgRwgZktNbM3zGxQHOOpTBd3zw2WtwNdKqlT2+etunEBNDezLDP7yMxqPYlVIYzzVV0OvGVm2WY2tZaPPZnIFYyKwj5fVcUFIZwvM/uJmW0BbgYqu+Qfl/OVdLo7kKq5+yoz+znwFnAQWAKU1oGYFhGZjuSAmV0J/AHoW7uRRri7m1nCDTU9SVw93T3HzPoAfzOzZe6+vjbjq2MuDs5XZ2Cema0O/pKPKzO7BygBno33sWJRjbhq/Xy5+z3APWb2Q+AO4L54Hq+cWlBx5u5Puftwd78U2At8WqFKDse3YLoHZaHF5O757n4gWP4L0MTMOsYzpgp2lF/mDN53VlKn1s9bNePC3XOC9w1E+l+GxTmu6gjjfFVL1PnaCbxK5PJaXJnZROBLwM0edKJUEMr5qkZcoZyvKM8C/1pJeVzOlxJUnAV/5WBmPYj09cytUOV1YIJFnA/sj7qMFEpMZtbVzCxYziDy/2RPPGOq4HWgfBRQJvBaJXXeBEabWbtgNN3ooCzUuIJ4mgXLHYGLgJVxjqs6FgJ9zay3mTUFxhH5PqEys1Zmlly+TOTfcfmJtzrtY44BfgB8xd0PVVGt1s9XdeIK6XxFXz25GlhdSbX4/DzGYySIXseNbnmPyC+opcDng7JpwLRg2YDHiIwYWsYJRjvVYkx3ACuCzz8CLoxjLM8BuUAxkevWU4AOwHxgLZFRhu2DuunA76K2nQysC16TEiEu4MLg33Fp8D6lFuK6Nlg+AuwA3gzqpgF/idr2SiKt5fXAPYkQF5FRckuD14paimsdkf6SJcHriQQ5XyeNK6Tz9TKRJPgJ8EegW8X/98F6jf88aiYJERFJSLrEJyIiCUkJSkREEpISlIiIJCQlKBERSUhKUCIikpCUoEREJCEpQYnUc8HjGU5pJhAzmxj9eIXT2ZdIrJSgROREJhK5UVSk1ilBidQSM+sVPJBulpl9ambPmtkoM/tn8JC3jOD1oZktNrMPzOzsYNvvmtmMYHmIRR442bKK43QIHiy3wsx+R2S2kvLPbjGzBcHD5540s8ZB+QEz+2WwzXwz62Rm1xOZLeDZoH6LYDffNLNFFnloXv94njNp2JSgRGrXWcAvgP7B6ybgYuBO4N+JzHN2ibsPI/JYg58G2/0vcJaZXQvMBL7mVc8jdx/wvrsPIjKZaA8AMxsA3Ahc5O5Dicxif3OwTSsgK9jmH8B97v57IIvIxKVD3f1wUHe3u58HPB7ELRIXetyGSO3a6O7LAMxsBTDf3d3MlgG9gDbA7GCCTgeaALh7WTDT9SfAk+7+zxMc41KCh1C6+5/NbG9Q/nlgOLAwmAu4BcdmZC8DXgiWnwFeOcH+yz/LJsSHXUr9pwQlUruORC2XRa2XEfl5fAD4u7tfa2a9iDyuo1xf4ACn3idkwGx3/2E16p5oks7ymEvR7xCJI13iE0ksbTj2HJ2J5YVm1gb4FZHWUYegf6gq7xK5dIiZfRFoF5TPB66PetxKezPrGXzWCCjf503A+8FyAZB8Gt9H5JQpQYkkloeAn5nZYo5vnfwSeMzdPyXyCIQHyxNNJf4TuDS4hHgd8BmAu68E7iXyuPBPgHlAarDNQSDDzJYDlwP3B+WzgCcqDJIQqRV63IaIYGYH3L112HGIRFMLSkREEpJaUCJ1lJlNAr5dofif7n57GPGI1DQlKBERSUi6xCciIglJCUpERBKSEpSIiCQkJSgREUlISlAiIpKQ/g8nyi5jh0y6HQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "plt.plot(max_depth_s, test_means)\n",
    "plt.xlabel( 'max_depth' )                                                                                    \n",
    "plt.ylabel( 'neg_log_loss' )\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/chenxiangkong/.local/lib/python3.6/site-packages/lightgbm/engine.py:430: UserWarning: Found `n_estimators` in params. Will use it instead of argument\n",
      "  warnings.warn(\"Found `{}` in params. Will use it instead of argument\".format(alias))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "random_forest- n_estimators: 344\n",
      "random_forest- cv score: 0.07222350898216418\n"
     ]
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'num_leaves': 70,\n",
    "          'max_depth': 13,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':1.0,\n",
    "          'subsample':1.0,\n",
    "          'learning_rate':0.0001\n",
    "         }\n",
    "\n",
    "cv_result = lgbm.cv(params,lgbmtrain,num_boost_round=10000,nfold=5,metrics='binary_logloss',early_stopping_rounds=10,categorical_feature=categorical_feat,seed=7)\n",
    "print('random_forest- n_estimators:' , len(cv_result['binary_logloss-mean']))\n",
    "print('random_forest- cv score:' , cv_result['binary_logloss-mean'][-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "进过调优，基本确定所以主要参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(boosting='gbdt', boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', is_unbalance=True,\n",
       "        learning_rate=0.0001, max_depth=13, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_data_in_leaf=20, min_split_gain=0.0,\n",
       "        n_estimators=344, n_jobs=5, num_leaves=70, objective='binary',\n",
       "        random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=False,\n",
       "        subsample=1.0, subsample_for_bin=200000, subsample_freq=0)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "params = {'boosting': 'gbdt',\n",
    "          'objective': 'binary',\n",
    "          'is_unbalance':True, \n",
    "          'n_jobs': 5,\n",
    "          'num_leaves': 70,\n",
    "          'max_depth': 13,\n",
    "          'min_data_in_leaf':20,\n",
    "          'min_child_samples':20,\n",
    "          'colsample_bytree':1.0,\n",
    "          'subsample':1.0,\n",
    "          'learning_rate':0.0001,\n",
    "          'n_estimators': 344\n",
    "         }\n",
    "\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "lgc=lg.fit(X_train, Y_train)\n",
    "lgc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 166, 2312, 3141, 1091,  358,  789, 2030, 2047,   10,  782,  508,\n",
       "       1293,  414, 1562, 1045, 1121,   78,   88,  362, 1020,  135, 1174,\n",
       "       2210])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importances=lgc.feature_importances_\n",
    "importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Gender', 'City', 'Monthly_Income', 'Loan_Amount_Applied',\n",
       "       'Loan_Tenure_Applied', 'Existing_EMI', 'Employer_Name',\n",
       "       'Salary_Account', 'Mobile_Verified', 'Var5', 'Var1',\n",
       "       'Loan_Amount_Submitted', 'Loan_Tenure_Submitted', 'Interest_Rate',\n",
       "       'Processing_Fee', 'EMI_Loan_Submitted', 'Filled_Form', 'Device_Type',\n",
       "       'Var2', 'Source', 'Var4', 'Employer_Name_Count', 'age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Monthly_Income', 3141),\n",
       " ('City', 2312),\n",
       " ('age', 2210),\n",
       " ('Salary_Account', 2047),\n",
       " ('Employer_Name', 2030),\n",
       " ('Interest_Rate', 1562),\n",
       " ('Loan_Amount_Submitted', 1293),\n",
       " ('Employer_Name_Count', 1174),\n",
       " ('EMI_Loan_Submitted', 1121),\n",
       " ('Loan_Amount_Applied', 1091),\n",
       " ('Processing_Fee', 1045),\n",
       " ('Source', 1020),\n",
       " ('Existing_EMI', 789),\n",
       " ('Var5', 782),\n",
       " ('Var1', 508),\n",
       " ('Loan_Tenure_Submitted', 414),\n",
       " ('Var2', 362),\n",
       " ('Loan_Tenure_Applied', 358),\n",
       " ('Gender', 166),\n",
       " ('Var4', 135),\n",
       " ('Device_Type', 88),\n",
       " ('Filled_Form', 78),\n",
       " ('Mobile_Verified', 10)]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dic = {}\n",
    "\n",
    "for index, feature in enumerate(feat_names):\n",
    "    dic.update({feature: importances[index]})\n",
    "    \n",
    "feature_importances_=sorted(dic.items(),key = lambda x:x[1],reverse = True)\n",
    "feature_importances_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "重要特征topN:Monthly_Income -> City -> age -> Salary_Account -> Employer_Name -> Interest_Rate -> Loan_Amount_Submitted -> Employer_Name_Count"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
